import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression

from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_absolute_error

# Download the Data

gold_train = pd.read_csv('/datasets/gold_recovery_train.csv')
gold_test = pd.read_csv('/datasets/gold_recovery_test.csv')
gold_full = pd.read_csv('/datasets/gold_recovery_full.csv')

# View the data

display(gold_train)
display(gold_test)
display(gold_full)

# Understand the data
gold_train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 16860 entries, 0 to 16859
Data columns (total 87 columns):
 #   Column                                              Non-Null Count  Dtype  
---  ------                                              --------------  -----  
 0   date                                                16860 non-null  object 
 1   final.output.concentrate_ag                         16788 non-null  float64
 2   final.output.concentrate_pb                         16788 non-null  float64
 3   final.output.concentrate_sol                        16490 non-null  float64
 4   final.output.concentrate_au                         16789 non-null  float64
 5   final.output.recovery                               15339 non-null  float64
 6   final.output.tail_ag                                16794 non-null  float64
 7   final.output.tail_pb                                16677 non-null  float64
 8   final.output.tail_sol                               16715 non-null  float64
 9   final.output.tail_au                                16794 non-null  float64
 10  primary_cleaner.input.sulfate                       15553 non-null  float64
 11  primary_cleaner.input.depressant                    15598 non-null  float64
 12  primary_cleaner.input.feed_size                     16860 non-null  float64
 13  primary_cleaner.input.xanthate                      15875 non-null  float64
 14  primary_cleaner.output.concentrate_ag               16778 non-null  float64
 15  primary_cleaner.output.concentrate_pb               16502 non-null  float64
 16  primary_cleaner.output.concentrate_sol              16224 non-null  float64
 17  primary_cleaner.output.concentrate_au               16778 non-null  float64
 18  primary_cleaner.output.tail_ag                      16777 non-null  float64
 19  primary_cleaner.output.tail_pb                      16761 non-null  float64
 20  primary_cleaner.output.tail_sol                     16579 non-null  float64
 21  primary_cleaner.output.tail_au                      16777 non-null  float64
 22  primary_cleaner.state.floatbank8_a_air              16820 non-null  float64
 23  primary_cleaner.state.floatbank8_a_level            16827 non-null  float64
 24  primary_cleaner.state.floatbank8_b_air              16820 non-null  float64
 25  primary_cleaner.state.floatbank8_b_level            16833 non-null  float64
 26  primary_cleaner.state.floatbank8_c_air              16822 non-null  float64
 27  primary_cleaner.state.floatbank8_c_level            16833 non-null  float64
 28  primary_cleaner.state.floatbank8_d_air              16821 non-null  float64
 29  primary_cleaner.state.floatbank8_d_level            16833 non-null  float64
 30  rougher.calculation.sulfate_to_au_concentrate       16833 non-null  float64
 31  rougher.calculation.floatbank10_sulfate_to_au_feed  16833 non-null  float64
 32  rougher.calculation.floatbank11_sulfate_to_au_feed  16833 non-null  float64
 33  rougher.calculation.au_pb_ratio                     15618 non-null  float64
 34  rougher.input.feed_ag                               16778 non-null  float64
 35  rougher.input.feed_pb                               16632 non-null  float64
 36  rougher.input.feed_rate                             16347 non-null  float64
 37  rougher.input.feed_size                             16443 non-null  float64
 38  rougher.input.feed_sol                              16568 non-null  float64
 39  rougher.input.feed_au                               16777 non-null  float64
 40  rougher.input.floatbank10_sulfate                   15816 non-null  float64
 41  rougher.input.floatbank10_xanthate                  16514 non-null  float64
 42  rougher.input.floatbank11_sulfate                   16237 non-null  float64
 43  rougher.input.floatbank11_xanthate                  14956 non-null  float64
 44  rougher.output.concentrate_ag                       16778 non-null  float64
 45  rougher.output.concentrate_pb                       16778 non-null  float64
 46  rougher.output.concentrate_sol                      16698 non-null  float64
 47  rougher.output.concentrate_au                       16778 non-null  float64
 48  rougher.output.recovery                             14287 non-null  float64
 49  rougher.output.tail_ag                              14610 non-null  float64
 50  rougher.output.tail_pb                              16778 non-null  float64
 51  rougher.output.tail_sol                             14611 non-null  float64
 52  rougher.output.tail_au                              14611 non-null  float64
 53  rougher.state.floatbank10_a_air                     16807 non-null  float64
 54  rougher.state.floatbank10_a_level                   16807 non-null  float64
 55  rougher.state.floatbank10_b_air                     16807 non-null  float64
 56  rougher.state.floatbank10_b_level                   16807 non-null  float64
 57  rougher.state.floatbank10_c_air                     16807 non-null  float64
 58  rougher.state.floatbank10_c_level                   16814 non-null  float64
 59  rougher.state.floatbank10_d_air                     16802 non-null  float64
 60  rougher.state.floatbank10_d_level                   16809 non-null  float64
 61  rougher.state.floatbank10_e_air                     16257 non-null  float64
 62  rougher.state.floatbank10_e_level                   16809 non-null  float64
 63  rougher.state.floatbank10_f_air                     16802 non-null  float64
 64  rougher.state.floatbank10_f_level                   16802 non-null  float64
 65  secondary_cleaner.output.tail_ag                    16776 non-null  float64
 66  secondary_cleaner.output.tail_pb                    16764 non-null  float64
 67  secondary_cleaner.output.tail_sol                   14874 non-null  float64
 68  secondary_cleaner.output.tail_au                    16778 non-null  float64
 69  secondary_cleaner.state.floatbank2_a_air            16497 non-null  float64
 70  secondary_cleaner.state.floatbank2_a_level          16751 non-null  float64
 71  secondary_cleaner.state.floatbank2_b_air            16705 non-null  float64
 72  secondary_cleaner.state.floatbank2_b_level          16748 non-null  float64
 73  secondary_cleaner.state.floatbank3_a_air            16763 non-null  float64
 74  secondary_cleaner.state.floatbank3_a_level          16747 non-null  float64
 75  secondary_cleaner.state.floatbank3_b_air            16752 non-null  float64
 76  secondary_cleaner.state.floatbank3_b_level          16750 non-null  float64
 77  secondary_cleaner.state.floatbank4_a_air            16731 non-null  float64
 78  secondary_cleaner.state.floatbank4_a_level          16747 non-null  float64
 79  secondary_cleaner.state.floatbank4_b_air            16768 non-null  float64
 80  secondary_cleaner.state.floatbank4_b_level          16767 non-null  float64
 81  secondary_cleaner.state.floatbank5_a_air            16775 non-null  float64
 82  secondary_cleaner.state.floatbank5_a_level          16775 non-null  float64
 83  secondary_cleaner.state.floatbank5_b_air            16775 non-null  float64
 84  secondary_cleaner.state.floatbank5_b_level          16776 non-null  float64
 85  secondary_cleaner.state.floatbank6_a_air            16757 non-null  float64
 86  secondary_cleaner.state.floatbank6_a_level          16775 non-null  float64
dtypes: float64(86), object(1)
memory usage: 11.2+ MB

# Check that recovery is calculated correctly, from the training set, using `rougher.output.recovery`

# Define your rougher recovery function
def rougher_recovery(df):
    """Calculate recovery for a given dataframe."""

    # extract needed columns

    c = df['rougher.output.concentrate_au'] 
    f = df['rougher.input.feed_au'] 
    t = df['rougher.output.tail_au'] 
  


    # recovery formula
    rougher_recovery = ((c * (f - t)) / (f * (c - t))) * 100

    return rougher_recovery

display(rougher_recovery(gold_train))

# Define your final recovery function

def final_recovery(df):
    """Calculate recovery for a given dataframe."""

    # extract needed columns

    c1 = df['final.output.concentrate_au']
    f1 = df['rougher.output.concentrate_au']
    t1 = df['final.output.tail_au']


    # recovery formula
    final_recovery = ((c1 * (f1 - t1)) / (f1 * (c1 - t1))) * 100

    return final_recovery

display(final_recovery(gold_train))

0        87.107763
1        86.843261
2        86.842308
3        87.226430
4        86.688794
           ...    
16855    89.574376
16856    87.724007
16857    88.890579
16858    89.858126
16859    89.514960
Length: 16860, dtype: float64

0        93.944554
1        93.790501
2        93.509750
3        93.595268
4        93.811976
           ...    
16855    94.886132
16856    94.507593
16857    92.593426
16858    94.268532
16859    95.048379
Length: 16860, dtype: float64

# Define a function that shows how many correct/True or incorrect/False recovery calculations there are:

def bool_recovery(df):

    boolean = df['rougher.output.recovery'] == rougher_recovery(df)
    bool_count = boolean.value_counts()
    boolean1 = df['final.output.recovery'] == final_recovery(df)
    bool_count1 = boolean1.value_counts()

    return print(f"The Rougher Recovery Calculations Are: \n{bool_count}\n\nThe Final Recovery Calculations Are: \n{bool_count1}")


bool_recovery(gold_train)

The Rougher Recovery Calculations Are: 
False    10002
True      6858
dtype: int64

The Final Recovery Calculations Are: 
False    16314
True       546
dtype: int64

# Get the MAE values for the predicted (rougher_recovery()) and the actual (gold_train['rougher.output.recovery']) values.

predicted = rougher_recovery(gold_train)
actual = gold_train['rougher.output.recovery']

display(len(gold_train))
display(predicted.isna().sum())
display(actual.isna().sum())

# Fix the missing values to calculate MAE

# 1. Combine into one DF
pred_act_df = pd.DataFrame({'predicted_values_train': predicted, 'actual_values_train': actual})
display(pred_act_df)

# 2. Drop the NaN values
pred_act_df = pred_act_df.dropna()
display(pred_act_df)

# 3. Compute MAE
mae = mean_absolute_error(pred_act_df['actual_values_train'], pred_act_df['predicted_values_train'])
print(f"MAE for Rougher Recovery: {mae}")

16860

2283

2573

MAE for Rougher Recovery: 9.303415616264301e-15

# View the non-equal values
display(pred_act_df[pred_act_df['predicted_values_train'] != pred_act_df['actual_values_train']])

gold_train['rougher.output.recovery'].isna().sum()

2573

# View the non-equal number that differ more than 1e^-20
display(np.isclose(pred_act_df['predicted_values_train'], pred_act_df['actual_values_train'], atol=1e-20).sum())

14287

# Look at the columns in the test set and the training set to analyze the missing columns in the test set

display(gold_test.info())
print()
display(gold_train.info())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5856 entries, 0 to 5855
Data columns (total 53 columns):
 #   Column                                      Non-Null Count  Dtype  
---  ------                                      --------------  -----  
 0   date                                        5856 non-null   object 
 1   primary_cleaner.input.sulfate               5554 non-null   float64
 2   primary_cleaner.input.depressant            5572 non-null   float64
 3   primary_cleaner.input.feed_size             5856 non-null   float64
 4   primary_cleaner.input.xanthate              5690 non-null   float64
 5   primary_cleaner.state.floatbank8_a_air      5840 non-null   float64
 6   primary_cleaner.state.floatbank8_a_level    5840 non-null   float64
 7   primary_cleaner.state.floatbank8_b_air      5840 non-null   float64
 8   primary_cleaner.state.floatbank8_b_level    5840 non-null   float64
 9   primary_cleaner.state.floatbank8_c_air      5840 non-null   float64
 10  primary_cleaner.state.floatbank8_c_level    5840 non-null   float64
 11  primary_cleaner.state.floatbank8_d_air      5840 non-null   float64
 12  primary_cleaner.state.floatbank8_d_level    5840 non-null   float64
 13  rougher.input.feed_ag                       5840 non-null   float64
 14  rougher.input.feed_pb                       5840 non-null   float64
 15  rougher.input.feed_rate                     5816 non-null   float64
 16  rougher.input.feed_size                     5834 non-null   float64
 17  rougher.input.feed_sol                      5789 non-null   float64
 18  rougher.input.feed_au                       5840 non-null   float64
 19  rougher.input.floatbank10_sulfate           5599 non-null   float64
 20  rougher.input.floatbank10_xanthate          5733 non-null   float64
 21  rougher.input.floatbank11_sulfate           5801 non-null   float64
 22  rougher.input.floatbank11_xanthate          5503 non-null   float64
 23  rougher.state.floatbank10_a_air             5839 non-null   float64
 24  rougher.state.floatbank10_a_level           5840 non-null   float64
 25  rougher.state.floatbank10_b_air             5839 non-null   float64
 26  rougher.state.floatbank10_b_level           5840 non-null   float64
 27  rougher.state.floatbank10_c_air             5839 non-null   float64
 28  rougher.state.floatbank10_c_level           5840 non-null   float64
 29  rougher.state.floatbank10_d_air             5839 non-null   float64
 30  rougher.state.floatbank10_d_level           5840 non-null   float64
 31  rougher.state.floatbank10_e_air             5839 non-null   float64
 32  rougher.state.floatbank10_e_level           5840 non-null   float64
 33  rougher.state.floatbank10_f_air             5839 non-null   float64
 34  rougher.state.floatbank10_f_level           5840 non-null   float64
 35  secondary_cleaner.state.floatbank2_a_air    5836 non-null   float64
 36  secondary_cleaner.state.floatbank2_a_level  5840 non-null   float64
 37  secondary_cleaner.state.floatbank2_b_air    5833 non-null   float64
 38  secondary_cleaner.state.floatbank2_b_level  5840 non-null   float64
 39  secondary_cleaner.state.floatbank3_a_air    5822 non-null   float64
 40  secondary_cleaner.state.floatbank3_a_level  5840 non-null   float64
 41  secondary_cleaner.state.floatbank3_b_air    5840 non-null   float64
 42  secondary_cleaner.state.floatbank3_b_level  5840 non-null   float64
 43  secondary_cleaner.state.floatbank4_a_air    5840 non-null   float64
 44  secondary_cleaner.state.floatbank4_a_level  5840 non-null   float64
 45  secondary_cleaner.state.floatbank4_b_air    5840 non-null   float64
 46  secondary_cleaner.state.floatbank4_b_level  5840 non-null   float64
 47  secondary_cleaner.state.floatbank5_a_air    5840 non-null   float64
 48  secondary_cleaner.state.floatbank5_a_level  5840 non-null   float64
 49  secondary_cleaner.state.floatbank5_b_air    5840 non-null   float64
 50  secondary_cleaner.state.floatbank5_b_level  5840 non-null   float64
 51  secondary_cleaner.state.floatbank6_a_air    5840 non-null   float64
 52  secondary_cleaner.state.floatbank6_a_level  5840 non-null   float64
dtypes: float64(52), object(1)
memory usage: 2.4+ MB

None

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 16860 entries, 0 to 16859
Data columns (total 87 columns):
 #   Column                                              Non-Null Count  Dtype  
---  ------                                              --------------  -----  
 0   date                                                16860 non-null  object 
 1   final.output.concentrate_ag                         16788 non-null  float64
 2   final.output.concentrate_pb                         16788 non-null  float64
 3   final.output.concentrate_sol                        16490 non-null  float64
 4   final.output.concentrate_au                         16789 non-null  float64
 5   final.output.recovery                               15339 non-null  float64
 6   final.output.tail_ag                                16794 non-null  float64
 7   final.output.tail_pb                                16677 non-null  float64
 8   final.output.tail_sol                               16715 non-null  float64
 9   final.output.tail_au                                16794 non-null  float64
 10  primary_cleaner.input.sulfate                       15553 non-null  float64
 11  primary_cleaner.input.depressant                    15598 non-null  float64
 12  primary_cleaner.input.feed_size                     16860 non-null  float64
 13  primary_cleaner.input.xanthate                      15875 non-null  float64
 14  primary_cleaner.output.concentrate_ag               16778 non-null  float64
 15  primary_cleaner.output.concentrate_pb               16502 non-null  float64
 16  primary_cleaner.output.concentrate_sol              16224 non-null  float64
 17  primary_cleaner.output.concentrate_au               16778 non-null  float64
 18  primary_cleaner.output.tail_ag                      16777 non-null  float64
 19  primary_cleaner.output.tail_pb                      16761 non-null  float64
 20  primary_cleaner.output.tail_sol                     16579 non-null  float64
 21  primary_cleaner.output.tail_au                      16777 non-null  float64
 22  primary_cleaner.state.floatbank8_a_air              16820 non-null  float64
 23  primary_cleaner.state.floatbank8_a_level            16827 non-null  float64
 24  primary_cleaner.state.floatbank8_b_air              16820 non-null  float64
 25  primary_cleaner.state.floatbank8_b_level            16833 non-null  float64
 26  primary_cleaner.state.floatbank8_c_air              16822 non-null  float64
 27  primary_cleaner.state.floatbank8_c_level            16833 non-null  float64
 28  primary_cleaner.state.floatbank8_d_air              16821 non-null  float64
 29  primary_cleaner.state.floatbank8_d_level            16833 non-null  float64
 30  rougher.calculation.sulfate_to_au_concentrate       16833 non-null  float64
 31  rougher.calculation.floatbank10_sulfate_to_au_feed  16833 non-null  float64
 32  rougher.calculation.floatbank11_sulfate_to_au_feed  16833 non-null  float64
 33  rougher.calculation.au_pb_ratio                     15618 non-null  float64
 34  rougher.input.feed_ag                               16778 non-null  float64
 35  rougher.input.feed_pb                               16632 non-null  float64
 36  rougher.input.feed_rate                             16347 non-null  float64
 37  rougher.input.feed_size                             16443 non-null  float64
 38  rougher.input.feed_sol                              16568 non-null  float64
 39  rougher.input.feed_au                               16777 non-null  float64
 40  rougher.input.floatbank10_sulfate                   15816 non-null  float64
 41  rougher.input.floatbank10_xanthate                  16514 non-null  float64
 42  rougher.input.floatbank11_sulfate                   16237 non-null  float64
 43  rougher.input.floatbank11_xanthate                  14956 non-null  float64
 44  rougher.output.concentrate_ag                       16778 non-null  float64
 45  rougher.output.concentrate_pb                       16778 non-null  float64
 46  rougher.output.concentrate_sol                      16698 non-null  float64
 47  rougher.output.concentrate_au                       16778 non-null  float64
 48  rougher.output.recovery                             14287 non-null  float64
 49  rougher.output.tail_ag                              14610 non-null  float64
 50  rougher.output.tail_pb                              16778 non-null  float64
 51  rougher.output.tail_sol                             14611 non-null  float64
 52  rougher.output.tail_au                              14611 non-null  float64
 53  rougher.state.floatbank10_a_air                     16807 non-null  float64
 54  rougher.state.floatbank10_a_level                   16807 non-null  float64
 55  rougher.state.floatbank10_b_air                     16807 non-null  float64
 56  rougher.state.floatbank10_b_level                   16807 non-null  float64
 57  rougher.state.floatbank10_c_air                     16807 non-null  float64
 58  rougher.state.floatbank10_c_level                   16814 non-null  float64
 59  rougher.state.floatbank10_d_air                     16802 non-null  float64
 60  rougher.state.floatbank10_d_level                   16809 non-null  float64
 61  rougher.state.floatbank10_e_air                     16257 non-null  float64
 62  rougher.state.floatbank10_e_level                   16809 non-null  float64
 63  rougher.state.floatbank10_f_air                     16802 non-null  float64
 64  rougher.state.floatbank10_f_level                   16802 non-null  float64
 65  secondary_cleaner.output.tail_ag                    16776 non-null  float64
 66  secondary_cleaner.output.tail_pb                    16764 non-null  float64
 67  secondary_cleaner.output.tail_sol                   14874 non-null  float64
 68  secondary_cleaner.output.tail_au                    16778 non-null  float64
 69  secondary_cleaner.state.floatbank2_a_air            16497 non-null  float64
 70  secondary_cleaner.state.floatbank2_a_level          16751 non-null  float64
 71  secondary_cleaner.state.floatbank2_b_air            16705 non-null  float64
 72  secondary_cleaner.state.floatbank2_b_level          16748 non-null  float64
 73  secondary_cleaner.state.floatbank3_a_air            16763 non-null  float64
 74  secondary_cleaner.state.floatbank3_a_level          16747 non-null  float64
 75  secondary_cleaner.state.floatbank3_b_air            16752 non-null  float64
 76  secondary_cleaner.state.floatbank3_b_level          16750 non-null  float64
 77  secondary_cleaner.state.floatbank4_a_air            16731 non-null  float64
 78  secondary_cleaner.state.floatbank4_a_level          16747 non-null  float64
 79  secondary_cleaner.state.floatbank4_b_air            16768 non-null  float64
 80  secondary_cleaner.state.floatbank4_b_level          16767 non-null  float64
 81  secondary_cleaner.state.floatbank5_a_air            16775 non-null  float64
 82  secondary_cleaner.state.floatbank5_a_level          16775 non-null  float64
 83  secondary_cleaner.state.floatbank5_b_air            16775 non-null  float64
 84  secondary_cleaner.state.floatbank5_b_level          16776 non-null  float64
 85  secondary_cleaner.state.floatbank6_a_air            16757 non-null  float64
 86  secondary_cleaner.state.floatbank6_a_level          16775 non-null  float64
dtypes: float64(86), object(1)
memory usage: 11.2+ MB

None

# Display the number of columns in the training set and test set

display(len(gold_train.columns))
display(len(gold_test.columns))

87

53

# Missing values in the Training Set

with pd.option_context('display.max_rows', None):
    missing_values_train = gold_train.isna().sum()
    display(missing_values_train)

date                                                     0
final.output.concentrate_ag                             72
final.output.concentrate_pb                             72
final.output.concentrate_sol                           370
final.output.concentrate_au                             71
final.output.recovery                                 1521
final.output.tail_ag                                    66
final.output.tail_pb                                   183
final.output.tail_sol                                  145
final.output.tail_au                                    66
primary_cleaner.input.sulfate                         1307
primary_cleaner.input.depressant                      1262
primary_cleaner.input.feed_size                          0
primary_cleaner.input.xanthate                         985
primary_cleaner.output.concentrate_ag                   82
primary_cleaner.output.concentrate_pb                  358
primary_cleaner.output.concentrate_sol                 636
primary_cleaner.output.concentrate_au                   82
primary_cleaner.output.tail_ag                          83
primary_cleaner.output.tail_pb                          99
primary_cleaner.output.tail_sol                        281
primary_cleaner.output.tail_au                          83
primary_cleaner.state.floatbank8_a_air                  40
primary_cleaner.state.floatbank8_a_level                33
primary_cleaner.state.floatbank8_b_air                  40
primary_cleaner.state.floatbank8_b_level                27
primary_cleaner.state.floatbank8_c_air                  38
primary_cleaner.state.floatbank8_c_level                27
primary_cleaner.state.floatbank8_d_air                  39
primary_cleaner.state.floatbank8_d_level                27
rougher.calculation.sulfate_to_au_concentrate           27
rougher.calculation.floatbank10_sulfate_to_au_feed      27
rougher.calculation.floatbank11_sulfate_to_au_feed      27
rougher.calculation.au_pb_ratio                       1242
rougher.input.feed_ag                                   82
rougher.input.feed_pb                                  228
rougher.input.feed_rate                                513
rougher.input.feed_size                                417
rougher.input.feed_sol                                 292
rougher.input.feed_au                                   83
rougher.input.floatbank10_sulfate                     1044
rougher.input.floatbank10_xanthate                     346
rougher.input.floatbank11_sulfate                      623
rougher.input.floatbank11_xanthate                    1904
rougher.output.concentrate_ag                           82
rougher.output.concentrate_pb                           82
rougher.output.concentrate_sol                         162
rougher.output.concentrate_au                           82
rougher.output.recovery                               2573
rougher.output.tail_ag                                2250
rougher.output.tail_pb                                  82
rougher.output.tail_sol                               2249
rougher.output.tail_au                                2249
rougher.state.floatbank10_a_air                         53
rougher.state.floatbank10_a_level                       53
rougher.state.floatbank10_b_air                         53
rougher.state.floatbank10_b_level                       53
rougher.state.floatbank10_c_air                         53
rougher.state.floatbank10_c_level                       46
rougher.state.floatbank10_d_air                         58
rougher.state.floatbank10_d_level                       51
rougher.state.floatbank10_e_air                        603
rougher.state.floatbank10_e_level                       51
rougher.state.floatbank10_f_air                         58
rougher.state.floatbank10_f_level                       58
secondary_cleaner.output.tail_ag                        84
secondary_cleaner.output.tail_pb                        96
secondary_cleaner.output.tail_sol                     1986
secondary_cleaner.output.tail_au                        82
secondary_cleaner.state.floatbank2_a_air               363
secondary_cleaner.state.floatbank2_a_level             109
secondary_cleaner.state.floatbank2_b_air               155
secondary_cleaner.state.floatbank2_b_level             112
secondary_cleaner.state.floatbank3_a_air                97
secondary_cleaner.state.floatbank3_a_level             113
secondary_cleaner.state.floatbank3_b_air               108
secondary_cleaner.state.floatbank3_b_level             110
secondary_cleaner.state.floatbank4_a_air               129
secondary_cleaner.state.floatbank4_a_level             113
secondary_cleaner.state.floatbank4_b_air                92
secondary_cleaner.state.floatbank4_b_level              93
secondary_cleaner.state.floatbank5_a_air                85
secondary_cleaner.state.floatbank5_a_level              85
secondary_cleaner.state.floatbank5_b_air                85
secondary_cleaner.state.floatbank5_b_level              84
secondary_cleaner.state.floatbank6_a_air               103
secondary_cleaner.state.floatbank6_a_level              85
dtype: int64

# Missing values in the Test Set

missing_values_test = gold_test.isna().sum()

# Find the columns not in the test set

missing_test_columns = set(gold_train.columns) - set(gold_test.columns)
missing_test_columns

{'final.output.concentrate_ag',
 'final.output.concentrate_au',
 'final.output.concentrate_pb',
 'final.output.concentrate_sol',
 'final.output.recovery',
 'final.output.tail_ag',
 'final.output.tail_au',
 'final.output.tail_pb',
 'final.output.tail_sol',
 'primary_cleaner.output.concentrate_ag',
 'primary_cleaner.output.concentrate_au',
 'primary_cleaner.output.concentrate_pb',
 'primary_cleaner.output.concentrate_sol',
 'primary_cleaner.output.tail_ag',
 'primary_cleaner.output.tail_au',
 'primary_cleaner.output.tail_pb',
 'primary_cleaner.output.tail_sol',
 'rougher.calculation.au_pb_ratio',
 'rougher.calculation.floatbank10_sulfate_to_au_feed',
 'rougher.calculation.floatbank11_sulfate_to_au_feed',
 'rougher.calculation.sulfate_to_au_concentrate',
 'rougher.output.concentrate_ag',
 'rougher.output.concentrate_au',
 'rougher.output.concentrate_pb',
 'rougher.output.concentrate_sol',
 'rougher.output.recovery',
 'rougher.output.tail_ag',
 'rougher.output.tail_au',
 'rougher.output.tail_pb',
 'rougher.output.tail_sol',
 'secondary_cleaner.output.tail_ag',
 'secondary_cleaner.output.tail_au',
 'secondary_cleaner.output.tail_pb',
 'secondary_cleaner.output.tail_sol'}

# Confirm the number of missing test columns

len(missing_test_columns)

34

# Missing Data

# Training Set
missing_values_train

# Test Set
missing_values_test

# Full Dataset
missing_values_full = gold_full.isna().sum()

# Full Set missing data
with pd.option_context('display.max_rows', None):
    display(missing_values_full)

date                                                     0
final.output.concentrate_ag                             89
final.output.concentrate_pb                             87
final.output.concentrate_sol                           385
final.output.concentrate_au                             86
final.output.recovery                                 1963
final.output.tail_ag                                    83
final.output.tail_pb                                   200
final.output.tail_sol                                  271
final.output.tail_au                                    81
primary_cleaner.input.sulfate                         1609
primary_cleaner.input.depressant                      1546
primary_cleaner.input.feed_size                          0
primary_cleaner.input.xanthate                        1151
primary_cleaner.output.concentrate_ag                   98
primary_cleaner.output.concentrate_pb                  448
primary_cleaner.output.concentrate_sol                 798
primary_cleaner.output.concentrate_au                   98
primary_cleaner.output.tail_ag                         102
primary_cleaner.output.tail_pb                         122
primary_cleaner.output.tail_sol                        351
primary_cleaner.output.tail_au                          99
primary_cleaner.state.floatbank8_a_air                  56
primary_cleaner.state.floatbank8_a_level                49
primary_cleaner.state.floatbank8_b_air                  56
primary_cleaner.state.floatbank8_b_level                43
primary_cleaner.state.floatbank8_c_air                  54
primary_cleaner.state.floatbank8_c_level                43
primary_cleaner.state.floatbank8_d_air                  55
primary_cleaner.state.floatbank8_d_level                43
rougher.calculation.sulfate_to_au_concentrate           44
rougher.calculation.floatbank10_sulfate_to_au_feed      44
rougher.calculation.floatbank11_sulfate_to_au_feed      44
rougher.calculation.au_pb_ratio                       1627
rougher.input.feed_ag                                   98
rougher.input.feed_pb                                  244
rougher.input.feed_rate                                553
rougher.input.feed_size                                439
rougher.input.feed_sol                                 359
rougher.input.feed_au                                   99
rougher.input.floatbank10_sulfate                     1301
rougher.input.floatbank10_xanthate                     469
rougher.input.floatbank11_sulfate                      678
rougher.input.floatbank11_xanthate                    2257
rougher.output.concentrate_ag                           98
rougher.output.concentrate_pb                           98
rougher.output.concentrate_sol                         190
rougher.output.concentrate_au                           98
rougher.output.recovery                               3119
rougher.output.tail_ag                                2737
rougher.output.tail_pb                                  98
rougher.output.tail_sol                               2736
rougher.output.tail_au                                2736
rougher.state.floatbank10_a_air                         70
rougher.state.floatbank10_a_level                       69
rougher.state.floatbank10_b_air                         70
rougher.state.floatbank10_b_level                       69
rougher.state.floatbank10_c_air                         70
rougher.state.floatbank10_c_level                       62
rougher.state.floatbank10_d_air                         75
rougher.state.floatbank10_d_level                       67
rougher.state.floatbank10_e_air                        620
rougher.state.floatbank10_e_level                       67
rougher.state.floatbank10_f_air                         75
rougher.state.floatbank10_f_level                       74
secondary_cleaner.output.tail_ag                       100
secondary_cleaner.output.tail_pb                       116
secondary_cleaner.output.tail_sol                     2215
secondary_cleaner.output.tail_au                        98
secondary_cleaner.state.floatbank2_a_air               383
secondary_cleaner.state.floatbank2_a_level             125
secondary_cleaner.state.floatbank2_b_air               178
secondary_cleaner.state.floatbank2_b_level             128
secondary_cleaner.state.floatbank3_a_air               131
secondary_cleaner.state.floatbank3_a_level             129
secondary_cleaner.state.floatbank3_b_air               124
secondary_cleaner.state.floatbank3_b_level             126
secondary_cleaner.state.floatbank4_a_air               145
secondary_cleaner.state.floatbank4_a_level             129
secondary_cleaner.state.floatbank4_b_air               108
secondary_cleaner.state.floatbank4_b_level             109
secondary_cleaner.state.floatbank5_a_air               101
secondary_cleaner.state.floatbank5_a_level             101
secondary_cleaner.state.floatbank5_b_air               101
secondary_cleaner.state.floatbank5_b_level             100
secondary_cleaner.state.floatbank6_a_air               119
secondary_cleaner.state.floatbank6_a_level             101
dtype: int64

# Get the percent & count of missing values for each column (Full Dataset)
percent_missing_full = (missing_values_full.values / 22716) * 100

missing_values_full_df = pd.DataFrame(missing_values_full, columns=['miss_cnt'])
percent_missing_full_df = pd.DataFrame(percent_missing_full, 
                                       index=missing_values_full.index, 
                                       columns=['pct_missing'])

missing_values_full_copy = missing_values_full_df.copy()

missing_values_full_copy = pd.concat([missing_values_full_copy, percent_missing_full_df],axis = 1)
missing_cp_full = missing_values_full_copy

# Get the percent & count of missing values for each column (Training set)
percent_missing_train = (missing_values_train.values / 16860) * 100

missing_values_train_df = pd.DataFrame(missing_values_train, columns = ['miss_cnt_train'])
percent_missing_train_df = pd.DataFrame(percent_missing_train,
                                        index = missing_values_train.index,
                                       columns = ['pct_missing_train'])

missing_cp_train = pd.concat([missing_values_train_df,percent_missing_train_df],axis = 1)
missing_cp_train

# Get the percent & count of missing values for each column (Test set)
percent_missing_test = (missing_values_test.values / 5836) * 100

missing_values_test_df = pd.DataFrame(missing_values_test, columns = ['miss_cnt_test'])
percent_missing_test_df = pd.DataFrame(percent_missing_test,
                                      index = missing_values_test.index,
                                      columns = ['pct_missing_test'])

missing_cp_test = pd.concat([missing_values_test_df,percent_missing_test_df], axis = 1)
display(missing_cp_test)

# Compare missing percentages
combined_missing = pd.concat([missing_cp_full,missing_cp_train, missing_cp_test], axis = 1)

combined_missing = combined_missing.round(3)

with pd.option_context('display.max_rows', None):
    display(combined_missing)

# Using this line of code to find percentage in specific categories <1%, 1-<5%, and so on

with pd.option_context('display.max_rows', None):
    display(combined_missing[combined_missing[['pct_missing','pct_missing_train', 'pct_missing_test']] >= 15.0])

# Get rid of the target Nan values `

# Training Dataset 

# View the NaN values for final.output.recovery (1,521 - 9.06% of data)
display(gold_train['final.output.recovery'].isna().sum())

# Drop the NaN values in final.output.recovery
train_target_df = pd.DataFrame(gold_train['final.output.recovery'].dropna())
display(train_target_df)

# Combine the original DF with train_target_df
gold_train_new = train_target_df.join(gold_train.drop(columns = ['final.output.recovery']), how = 'left')
display(gold_train_new)

1521

# Training Dataset

# Look at the missing values and re-determine percentage
new_missing_count = gold_train_new.isna().sum()

# Separates and calculates the pct values
new_missing_count_array = (new_missing_count.values / 15339) * 100

# Creates a DF w/ just the index(columns in this case) with the missing count
new_missing_count_df = pd.DataFrame(new_missing_count, columns = ['missing_cnt'])

# Creates a DF w/ just the index(columns in this case) with the missing percent
new_missing_count_2 = pd.DataFrame(new_missing_count_array, index = new_missing_count_df.index, columns = ['missing_pct'])

# Combines the 2 DF's to show count and percent missing
new_missing_count_df = pd.concat([new_missing_count_df,new_missing_count_2], axis = 1)

# Look at missing_pct that is greater than or equal to 1 
new_missing_count_df[new_missing_count_df['missing_pct'] >= 1].sort_values(by = 'missing_pct',ascending = False)

# Get rid of the target Nan values

# Full Dataset 

# View the NaN values for final.output.recovery (1,963 - 8.64% of data)
display(gold_full['final.output.recovery'].isna().sum())

# Drop the NaN values in final.output.recovery
full_target_df = pd.DataFrame(gold_full['final.output.recovery'].dropna())

# Combine the original DF with full_target_df
gold_full_new = full_target_df.join(gold_full.drop(columns = ['final.output.recovery']), how = 'left')
display(gold_full_new)

1963

# Full Dataset

# Look at the missing values and re-determine percentage
new_missing_full_count = gold_full_new.isna().sum()

# Separates and calculates the pct values
new_missing_count_array_1 = (new_missing_full_count.values / 20753) * 100

# Creates a DF w/ just the index(columns in this case) with the missing count
new_missing_full_count_df = pd.DataFrame(new_missing_full_count, columns = ['missing_cnt'])

# Creates a DF w/ just the index(columns in this case) with the missing percent
new_missing_full_count_2 = pd.DataFrame(new_missing_count_array_1, index = new_missing_full_count_df.index, columns = ['missing_pct'])

# Combines the 2 DF's to show count and percent missing
new_missing_full_count_df = pd.concat([new_missing_full_count_df,new_missing_full_count_2], axis = 1)
display(new_missing_full_count_df)

# Look at missing_pct that is greater than or equal to 1 
new_missing_full_count_df[new_missing_full_count_df['missing_pct'] >= 1].sort_values(by = 'missing_pct',ascending = False)

gold_full[['final.output.recovery']]

# Full Dataset

# Drop All calculation columns from the Full Dataset

calculation_full = gold_full_new.filter(like="calculation",axis = 1)
gold_full_new1 = gold_full_new.drop(calculation_full, axis=1)

# Drop all output columns from the Full Dataset except for final.output.recovery and rougher.output.recovery
output_full = gold_full_new1.filter(like = "output", axis = 1)

# Make a DF w/ only final.output.recovery and rougher.output.recovery
final_full = gold_full_new1[['final.output.recovery']]

# Drop rougher_final from the output_train DF
output_full = output_full.drop(final_full, axis=1)

# Drop output_train from gold_train_new1 (the new Training Dataset made)
gold_full_new1 = gold_full_new1.drop(output_full, axis = 1)
gold_full_new1

# Move final.output.recovery 
col = 'final.output.recovery'
cols = list(gold_full_new1.columns)
cols.insert(1,cols.pop(cols.index(col)))
gold_full_new1 = gold_full_new1[cols]

gold_full_new1

# Training Dataset

# Drop All calculation columns from the Training Dataset

calculation_train = gold_train_new.filter(like="calculation",axis = 1)
gold_train_new1 = gold_train_new.drop(calculation_train, axis=1)

# Drop all output columns from the Training Dataset except for final.output.recovery and rougher.output.recovery
output_train = gold_train_new1.filter(like = "output", axis = 1)

# Make a DF w/ only final.output.recovery and rougher.output.recovery
final_train = gold_train_new1[['final.output.recovery']]

# Drop rougher_final from the output_train DF
output_train = output_train.drop(final_train, axis=1)

# Drop output_train from gold_train_new1 (the new Training Dataset made)
gold_train_new1 = gold_train_new1.drop(output_train, axis = 1)
gold_train_new1

# Move final.output.recovery 
col = 'final.output.recovery'
cols = list(gold_train_new1.columns)
cols.insert(1,cols.pop(cols.index(col)))
gold_train_new1 = gold_train_new1[cols]

gold_train_new1

# Training Dataset

# Look at the missing values and re-determine percentage
new_missing_train_count1 = gold_train_new1.isna().sum()

# Separates and calculate the pct values
new_missing_count_array1 = (new_missing_train_count1.values / 15339) * 100

# Create a DF w/ just the index(columns in this case) with the missing count
new_missing_train_count_df1 = pd.DataFrame(new_missing_train_count1, columns = ['missing_cnt'])

# Creates a DF w/ just the index(columns in this case) with the missing percent
new_missing_train_count2 = pd.DataFrame(new_missing_count_array1, index = new_missing_train_count_df1.index, columns = ['missing_pct'])

# Combines the 2 DF's to show count and percent missing
new_missing_train_count_df1 = pd.concat([new_missing_train_count_df1,new_missing_train_count2], axis = 1)
display(new_missing_train_count_df1)

# Find the NaN values that are missing more than 1%
new_missing_train_count_df1[new_missing_train_count_df1['missing_pct'] >= 1.0].sort_values(by='missing_pct', ascending = False)

# Full Dataset

# Look at the missing values and re-determine percentage
new_missing_full_count1 = gold_full_new1.isna().sum()

# Separates and calculates the pct values
new_missing_count_array2 = (new_missing_full_count1.values / 20753) * 100

# Creates a DF w/ just the index(columns in this case) with the missing count
new_missing_full_count_df1 = pd.DataFrame(new_missing_full_count1, columns = ['missing_cnt'])

# Creates a DF w/ just the index(columns in this case) with the missing percent
new_missing_full_count2 = pd.DataFrame(new_missing_count_array2, index = new_missing_full_count_df1.index, columns = ['missing_pct'])

# Combines the 2 DF's to show count and percent missing
new_missing_full_count_df1 = pd.concat([new_missing_full_count_df1,new_missing_full_count2], axis = 1)
display(new_missing_full_count_df1)

# Find the NaN values that are missing less than 1%
new_missing_full_count_df1[new_missing_full_count_df1['missing_pct'] >= 1.0].sort_values(by='missing_pct', ascending = False)

# Training Dataset

# Idendtify the rows where the NaN should be dropped
new_missing_train_count_df1[new_missing_train_count_df1['missing_pct'] < 1.0].sort_values(by = 'missing_pct', ascending = False)

# Training Dataset

# Get rid of the rows that have less than 1% of NaN values

index_train = new_missing_train_count_df1[new_missing_train_count_df1['missing_pct'] < 1.0].sort_values(
    by = 'missing_pct', ascending = False).index

# Turn the columns w/ less than 1% to a list
index_train = list(index_train)

# Drop the NaN values with less than 1% of NaN values
gold_train_new2 = gold_train_new1.dropna(subset = index_train)

# Look at the NaN values in the DF
gold_train_new2.isna().sum().sort_values(ascending = False)

rougher.input.floatbank11_xanthate            654
rougher.state.floatbank10_e_air               508
rougher.input.floatbank11_sulfate             262
primary_cleaner.input.sulfate                 259
rougher.input.floatbank10_sulfate             259
secondary_cleaner.state.floatbank2_a_air      220
primary_cleaner.input.xanthate                190
primary_cleaner.input.depressant              189
rougher.input.feed_rate                       183
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank2_a_level      0
rougher.state.floatbank10_e_level               0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
date                                            0
rougher.state.floatbank10_d_level               0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank3_b_air        0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_d_air                 0
rougher.input.feed_ag                           0
primary_cleaner.input.feed_size                 0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_pb                           0
rougher.state.floatbank10_c_level               0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
final.output.recovery                           0
rougher.state.floatbank10_c_air                 0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# All values are still above 1%; thus imputation is necessary for the rest of the NaN values

display(654 / 14855 * 100)
display(508 / 14855 * 100)
display(262 / 14855 * 100)
display(259 / 14855 * 100)
display(259 / 14855 * 100)
display(220 / 14855 * 100)
display(190 / 14855 * 100)
display(189 / 14855 * 100)
display(183 / 14855 * 100)

4.402558061258835

3.4197239986536516

1.7637159205654662

1.743520700100976

1.743520700100976

1.4809828340626052

1.2790306294177045

1.2722988892628744

1.2319084483338942

# Full Dataset

# Idendtify the rows where the NaN should be dropped

new_missing_full_count_df1[new_missing_full_count_df1['missing_pct'] < 1.0].sort_values(by = 'missing_pct', ascending = False)

# Full Dataset

# Get rid of the rows that have less than 1% of NaN values

index_full = new_missing_full_count_df1[new_missing_full_count_df1['missing_pct'] < 1.0].sort_values(
    by = 'missing_pct', ascending = False).index

# Turn the columns w/ less than 1% to a list
index_full = list(index_full)

# Drop the NaN values with less than 1% of NaN values
gold_full_new2 = gold_full_new1.dropna(subset = index_full)

# Look at the NaN values in the DF
gold_full_new2.isna().sum().sort_values(ascending = False)

rougher.input.floatbank11_xanthate            684
rougher.state.floatbank10_e_air               508
rougher.input.floatbank11_sulfate             272
primary_cleaner.input.sulfate                 263
rougher.input.floatbank10_sulfate             261
secondary_cleaner.state.floatbank2_a_air      223
primary_cleaner.input.xanthate                193
primary_cleaner.input.depressant              193
rougher.input.feed_rate                       185
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank2_a_level      0
rougher.state.floatbank10_e_level               0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
date                                            0
rougher.state.floatbank10_d_level               0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank3_b_air        0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_d_air                 0
rougher.input.feed_ag                           0
primary_cleaner.input.feed_size                 0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_pb                           0
rougher.state.floatbank10_c_level               0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
final.output.recovery                           0
rougher.state.floatbank10_c_air                 0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# Although some values are below 1%, to keep aligned with training set, these will also be imputated

display(684 / 20226 * 100)
display(508 / 20226 * 100)
display(272 / 20226 * 100)
display(263 / 20226 * 100)
display(261 / 20226 * 100)
display(223 / 20226 * 100)
display(193 / 20226 * 100)
display(193 / 20226 * 100)
display(185 / 20226 * 100)

3.3817858202313853

2.5116187085929003

1.3448037179867498

1.3003065361416

1.2904182735093443

1.1025412834964896

0.954217344012657

0.954217344012657

0.9146642934836349

# Training Dataset

# Investigate the 9 missing columns

imp_cols = ['rougher.input.floatbank11_xanthate','rougher.state.floatbank10_e_air','rougher.input.floatbank11_sulfate',
'primary_cleaner.input.sulfate','rougher.input.floatbank10_sulfate','secondary_cleaner.state.floatbank2_a_air',
'primary_cleaner.input.xanthate','primary_cleaner.input.depressant','rougher.input.feed_rate']	

# Create a histogram for the 9 features with missing values
gold_train_new2[imp_cols].hist(figsize = [15,10])
plt.show()

display(gold_train_new2[imp_cols].mean())
print()
display(gold_train_new2[imp_cols].median())
print()
display(gold_train_new2[imp_cols].describe())

rougher.input.floatbank11_xanthate             5.888175
rougher.state.floatbank10_e_air             1078.990959
rougher.input.floatbank11_sulfate             11.389359
primary_cleaner.input.sulfate                134.643263
rougher.input.floatbank10_sulfate             11.763041
secondary_cleaner.state.floatbank2_a_air      29.341060
primary_cleaner.input.xanthate                 0.879605
primary_cleaner.input.depressant               8.929893
rougher.input.feed_rate                      467.518312
dtype: float64

rougher.input.floatbank11_xanthate             5.999159
rougher.state.floatbank10_e_air             1049.988150
rougher.input.floatbank11_sulfate             11.414551
primary_cleaner.input.sulfate                134.315614
rougher.input.floatbank10_sulfate             11.708082
secondary_cleaner.state.floatbank2_a_air      30.017078
primary_cleaner.input.xanthate                 0.888980
primary_cleaner.input.depressant               8.043323
rougher.input.feed_rate                      498.403275
dtype: float64

# View columns that look similar to the missing features (rougher.input.floatbank##_xanthate) 

# Look for columns with rougher.input.floatbank and then drop the columns that do not have rougher.input.floatbank##_xanthate
xanthate = gold_train_new2.filter(like="rougher.input.floatbank", axis = 1).drop(
    gold_train_new2[['rougher.input.floatbank11_sulfate','rougher.input.floatbank10_sulfate']], axis = 1)

# Compare rougher.input.floatbank##_xanthate columns (subtract)
xanthate_difference = xanthate['rougher.input.floatbank10_xanthate'] - xanthate['rougher.input.floatbank11_xanthate']

# Create a DF to compare
xanthate['xanthate_difference'] = xanthate_difference



# Group the differences and compare statistics

print("Less Than -1")
display(xanthate[xanthate['xanthate_difference'] < -1].median())
display(xanthate[xanthate['xanthate_difference'] < -1].mean())
display(xanthate[xanthate['xanthate_difference'] < -1].max())
display(xanthate[xanthate['xanthate_difference'] < -1].min())
print(len(xanthate[xanthate['xanthate_difference'] <- 1]))

print()
print("Greater Than 1")
display(xanthate[xanthate['xanthate_difference'] > 1].median())
display(xanthate[xanthate['xanthate_difference'] > 1].mean())
display(xanthate[xanthate['xanthate_difference'] > 1].max())
display(xanthate[xanthate['xanthate_difference'] > 1].min())
display(len(xanthate[xanthate['xanthate_difference'] > 1]))

print()
print("Between 0 - 1")
display(xanthate[(xanthate['xanthate_difference'] > 0) & (xanthate['xanthate_difference'] <= 1)].median())
display(xanthate[(xanthate['xanthate_difference'] > 0) & (xanthate['xanthate_difference'] <= 1)].mean())
display(xanthate[(xanthate['xanthate_difference'] > 0) & (xanthate['xanthate_difference'] <= 1)].max())
display(xanthate[(xanthate['xanthate_difference'] > 0) & (xanthate['xanthate_difference'] <= 1)].min())
display(len(xanthate[(xanthate['xanthate_difference'] > 0) & (xanthate['xanthate_difference'] <= 1)]))

print()
print("Between 0 - (-)1")
display(xanthate[(xanthate['xanthate_difference'] < 0) & (xanthate['xanthate_difference'] >= -1)].median())
display(xanthate[(xanthate['xanthate_difference'] < 0) & (xanthate['xanthate_difference'] >= -1)].mean())
display(xanthate[(xanthate['xanthate_difference'] < 0) & (xanthate['xanthate_difference'] >= -1)].max())
display(xanthate[(xanthate['xanthate_difference'] < 0) & (xanthate['xanthate_difference'] >= -1)].min())
display(len(xanthate[(xanthate['xanthate_difference'] < 0) & (xanthate['xanthate_difference'] >= -1)]))
print()
print("All")
display(xanthate[xanthate['xanthate_difference'] > -20].median())
display(xanthate[xanthate['xanthate_difference'] > -20].mean())
display(xanthate[xanthate['xanthate_difference'] > -20].max())
display(xanthate[xanthate['xanthate_difference'] > -20].min())

Less Than -1

rougher.input.floatbank10_xanthate    6.243166
rougher.input.floatbank11_xanthate    7.493640
xanthate_difference                  -1.416374
dtype: float64

rougher.input.floatbank10_xanthate    4.437777
rougher.input.floatbank11_xanthate    6.770322
xanthate_difference                  -2.332545
dtype: float64

rougher.input.floatbank10_xanthate    7.624764
rougher.input.floatbank11_xanthate    8.833521
xanthate_difference                  -1.001298
dtype: float64

rougher.input.floatbank10_xanthate    0.000508
rougher.input.floatbank11_xanthate    1.482072
xanthate_difference                  -8.004946
dtype: float64

174

Greater Than 1

rougher.input.floatbank10_xanthate    5.781152
rougher.input.floatbank11_xanthate    1.931478
xanthate_difference                   3.203859
dtype: float64

rougher.input.floatbank10_xanthate    5.702966
rougher.input.floatbank11_xanthate    2.413444
xanthate_difference                   3.289522
dtype: float64

rougher.input.floatbank10_xanthate    8.036454
rougher.input.floatbank11_xanthate    6.318315
xanthate_difference                   7.576177
dtype: float64

rougher.input.floatbank10_xanthate    3.108537
rougher.input.floatbank11_xanthate    0.000290
xanthate_difference                   1.026330
dtype: float64

# Full Dataset

# Create a histogram for the 9 features with missing values
gold_full_new2[imp_cols].hist(figsize = [15,10])
plt.show()

display(gold_full_new2[imp_cols].mean())
print()
display(gold_full_new2[imp_cols].median())
print()
display(gold_full_new2[imp_cols].describe())

rougher.input.floatbank11_xanthate             6.058794
rougher.state.floatbank10_e_air             1072.584818
rougher.input.floatbank11_sulfate             12.069147
primary_cleaner.input.sulfate                145.346889
rougher.input.floatbank10_sulfate             12.314856
secondary_cleaner.state.floatbank2_a_air      28.535827
primary_cleaner.input.xanthate                 1.013894
primary_cleaner.input.depressant               8.863749
rougher.input.feed_rate                      473.234067
dtype: float64

rougher.input.floatbank11_xanthate             6.099767
rougher.state.floatbank10_e_air             1049.694499
rougher.input.floatbank11_sulfate             12.000267
primary_cleaner.input.sulfate                144.133440
rougher.input.floatbank10_sulfate             12.001148
secondary_cleaner.state.floatbank2_a_air      29.086048
primary_cleaner.input.xanthate                 0.942111
primary_cleaner.input.depressant               8.044753
rougher.input.feed_rate                      498.480665
dtype: float64

# View columns that look similar to the missing features (rougher.input.floatbank##_xanthate) - Full Dataset

# Look for columns with rougher.input.floatbank and then drop the columns that do not have rougher.input.floatbank##_xanthate

xanthate_full = gold_full_new2.filter(like="rougher.input.floatbank", axis = 1).drop(
    gold_full_new2[['rougher.input.floatbank11_sulfate','rougher.input.floatbank10_sulfate']], axis = 1)

# Compare rougher.input.floatbank##_xanthate columns (subtract)
xanthate_difference_full = xanthate_full['rougher.input.floatbank10_xanthate'] - xanthate_full['rougher.input.floatbank11_xanthate']

# Create a DF to compare
xanthate_full['xanthate_difference'] = xanthate_difference_full

# Check the DF's median difference to check if it aligns with the Training Dastaset
display(xanthate_full.median())
display(xanthate_full[xanthate_full['xanthate_difference'] > -20].median())

rougher.input.floatbank10_xanthate    6.004194
rougher.input.floatbank11_xanthate    6.099767
xanthate_difference                  -0.000087
dtype: float64

rougher.input.floatbank10_xanthate    6.098051
rougher.input.floatbank11_xanthate    6.099767
xanthate_difference                  -0.000087
dtype: float64

# Training Dataset

# Impute the 'rougher.input.floatbank11_xanthate' NaN values to match the coordinating 'rougher.input.floatbank10_xanthate' values

display(gold_train_new2[gold_train_new2['rougher.input.floatbank11_xanthate'].isna()])

# Fill the NaN values for floatbank 11 as the corresponding floatbank 10 values
gold_train_new2 = gold_train_new2.copy()

mask = gold_train_new2['rougher.input.floatbank11_xanthate']
gold_train_new2['rougher.input.floatbank11_xanthate'] = (mask.fillna(gold_train_new2['rougher.input.floatbank10_xanthate']))

# Ensure no more NaN values for rougher.input.floatbank11_xanthate
display(gold_train_new2[gold_train_new2['rougher.input.floatbank11_xanthate'].isna()])

# Test it, ensure correct values placed correctly
test = gold_train_new2[['rougher.input.floatbank11_xanthate','rougher.input.floatbank10_xanthate']]
test[2495:]

# Full Dataset

# Impute the 'rougher.input.floatbank11_xanthate' NaN values to match the coordinating 'rougher.input.floatbank10_xanthate' values

display(gold_full_new2[gold_full_new2['rougher.input.floatbank11_xanthate'].isna()])

# Fill the NaN values for floatbank 11 as the corresponding floatbank 10 values
gold_full_new2 = gold_full_new2.copy()

mask1 = gold_full_new2['rougher.input.floatbank11_xanthate']
gold_full_new2['rougher.input.floatbank11_xanthate'] = (mask1.fillna(gold_full_new2['rougher.input.floatbank10_xanthate']))

# Ensure no more NaN values for rougher.input.floatbank11_xanthate
display(gold_full_new2[gold_full_new2['rougher.input.floatbank11_xanthate'].isna()])

# Test it, ensure correct values placed correctly
test1 = gold_full_new2[['rougher.input.floatbank11_xanthate','rougher.input.floatbank10_xanthate']]
test1[2495:]

# Test Dataset

# Impute the 'rougher.input.floatbank11_xanthate' NaN values to match the coordinating 'rougher.input.floatbank10_xanthate' values

gold_test_new2 = gold_test

display(gold_test_new2[gold_test_new2['rougher.input.floatbank11_xanthate'].isna()])

# Fill the NaN values for floatbank 11 as the corresponding floatbank 10 values
mask_test_set = gold_test_new2['rougher.input.floatbank11_xanthate']
gold_test_new2['rougher.input.floatbank11_xanthate'] = (mask_test_set.fillna(gold_test_new2['rougher.input.floatbank10_xanthate']))


# Check how many rows were filled
with pd.option_context('display.max_columns',None):
    display(gold_test_new2[gold_test_new2['rougher.input.floatbank11_xanthate'].isna()])

# Compare the new histograms

gold_train_new2['rougher.input.floatbank11_xanthate'].hist(figsize = [8,5])
plt.title("Training Set")
plt.show()

gold_full_new2['rougher.input.floatbank11_xanthate'].hist(figsize = [8,5])
plt.title("Full Set")
plt.show()

# Show the histograms again, continue imputation

gold_train_new2[imp_cols].hist(figsize = [12,8])
plt.show()

test = gold_train_new2[['rougher.state.floatbank10_f_air','rougher.state.floatbank10_e_air']]

test[test.columns].hist(figsize = [10,4])
plt.show()

def r_state_fb_nan(df1):
    # Focus on the columns you are comparing
    df = df1.filter(like = 'rougher.state.floatbank', axis=1).copy()
    df_cols = df.filter(like = 'level').columns
    df_cols = list(df_cols)
    df = df.drop(df[df_cols], axis = 1)
    df = df.drop(['rougher.state.floatbank10_a_air','rougher.state.floatbank10_b_air','rougher.state.floatbank10_c_air','rougher.state.floatbank10_d_air'], axis = 1)
    
    # Get the difference of the columns and put it in a DF
    df_difference_f = df['rougher.state.floatbank10_f_air'] - df['rougher.state.floatbank10_e_air']
    df['difference_f'] = df_difference_f

    # Show the metrics of the entire DF
    print("Difference Metrics (fb10_e_air & fb10_f_air)")
    print("-------------------------------------------")
    print("Length")
    display(len(df['difference_f']))
    print()
    print("Median")
    display(df['difference_f'].median())
    print()
    print("Mean")
    display(df['difference_f'].mean())
    print()
    print("Std")
    display(df['difference_f'].std())
    print()
    print("Min")
    display(df['difference_f'].min())
    print()
    print("Max")
    display(df['difference_f'].max())
    print()
    print()

    # Metrics for Categories
    print("Metrics Difference: Categorical")
    print("-------------------------------------------")
    print()
    print("Difference F > 5")
    print("-------------------------------------------")
    print("Length")
    display(len(df[df['difference_f'] > 5]))
    print()
    print("Median")
    display(df[df['difference_f'] > 5].median())
    print()
    print("Mean")
    display(df[df['difference_f'] > 5].mean())
    print()
    print("Min")
    display(df[df['difference_f'] > 5].min())
    print()
    print("Max")
    display(df[df['difference_f'] > 5].max())
    print()
    print("Difference F Between 1 - 5")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] > 1) & (df['difference_f'] <= 5)]))
    print()
    print("Median")
    display(df[(df['difference_f'] > 1) & (df['difference_f'] <= 5)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] > 1) & (df['difference_f'] <= 5)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] > 1) & (df['difference_f'] <= 5)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] > 1) & (df['difference_f'] <= 5)].max())
    print()
    print("Difference F Between 0 - 1")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] > 0) & (df['difference_f'] <= 1)]))
    print()
    print("Median")
    display(df[(df['difference_f'] > 0) & (df['difference_f'] <= 1)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] > 0) & (df['difference_f'] <= 1)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] > 0) & (df['difference_f'] <= 1)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] > 0) & (df['difference_f'] <= 1)].max())
    print()
    print("Difference F Between (-)1 - 0")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -1) & (df['difference_f'] < 0)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -1) & (df['difference_f'] < 0)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -1) & (df['difference_f'] < 0)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -1) & (df['difference_f'] < 0)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -1) & (df['difference_f'] < 0)].max())
    print()
    print("Difference F Between (-)5 - (-)1")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -5) & (df['difference_f'] < -1)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -5) & (df['difference_f'] < -1)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -5) & (df['difference_f'] < -1)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -5) & (df['difference_f'] < -1)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -5) & (df['difference_f'] < -1)].max())
    print()
    print("Difference F Between (-)50 - (-)5")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -50) & (df['difference_f'] < -5)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -50) & (df['difference_f'] < -5)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -50) & (df['difference_f'] < -5)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -50) & (df['difference_f'] < -5)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -50) & (df['difference_f'] < -5)].max())
    print()
    print("Difference F Between (-)100 - (-)50")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -100) & (df['difference_f'] < -50)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -100) & (df['difference_f'] < -50)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -100) & (df['difference_f'] < -50)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -100) & (df['difference_f'] < -50)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -100) & (df['difference_f'] < -50)].max())
    print()
    print("Difference F Between (-)150 - (-)100")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -150) & (df['difference_f'] < -100)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -150) & (df['difference_f'] < -100)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -150) & (df['difference_f'] < -100)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -150) & (df['difference_f'] < -100)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -150) & (df['difference_f'] < -100)].max())
    print()
    print("Difference F Between (-)220 - (-)150")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] >= -220) & (df['difference_f'] < -150)]))
    print()
    print("Median")
    display(df[(df['difference_f'] >= -220) & (df['difference_f'] < -150)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] >= -220) & (df['difference_f'] < -150)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] >= -220) & (df['difference_f'] < -150)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] >= -220) & (df['difference_f'] < -150)].max())
    print()
    print("Difference F < (-)200")
    print("-------------------------------------------")
    print("Length")
    display(len(df[df['difference_f'] < -200]))
    print()
    print("Median")
    display(df[df['difference_f'] < -200].median())
    print()
    print("Mean")
    display(df[df['difference_f'] < -200].mean())
    print()
    print("Min")
    display(df[df['difference_f'] < -200].min())
    print()
    print("Max")
    display(df[df['difference_f'] < -200].max())
    print()
    display(df[df['difference_f'] ==0])
    print()
    print()
    print()
    print()
    print("Difference F > 0")
    print("-------------------------------------------")
    print("Length")
    display(len(df[df['difference_f'] > 0]))
    print()
    print("Median")
    display(df[df['difference_f'] > 0].median())
    print()
    print("Mean")
    display(df[df['difference_f'] > 0].mean())
    print()
    print("Min")
    display(df[df['difference_f'] > 0].min())
    print()
    print("Max")
    display(df[df['difference_f'] > 0].max())
    print()
    print()
    print("Difference F Between (-)50 - 0")
    print("-------------------------------------------")
    print("Length")
    display(len(df[(df['difference_f'] > -50) & (df['difference_f'] <= 0)]))
    print()
    print("Median")
    display(df[(df['difference_f'] > -50) & (df['difference_f'] <= 0)].median())
    print()
    print("Mean")
    display(df[(df['difference_f'] > -50) & (df['difference_f'] <= 0)].mean())
    print()
    print("Min")
    display(df[(df['difference_f'] > -50) & (df['difference_f'] <= 0)].min())
    print()
    print("Max")
    display(df[(df['difference_f'] > -50) & (df['difference_f'] <= 0)].max())
    print()
    print()
    print("Difference F < (-)50")
    print("-------------------------------------------")
    print("Length")
    display(len(df[df['difference_f'] <= -50]))
    print()
    print("Median")
    display(df[df['difference_f'] <= -50].median())
    print()
    print("Mean")
    display(df[df['difference_f'] <= -50].mean())
    print()
    print("Min")
    display(df[df['difference_f'] <= 50].min())
    print()
    print("Max")
    display(df[df['difference_f'] <= -50].max())

# All the missing rougher.state.floatbank10_e_air are between 844 and 856 in rougher.state.floatbank10_f_air (except 2)

r_state_fb = gold_train_new2[['rougher.state.floatbank10_e_air','rougher.state.floatbank10_f_air']].copy()
r_state_fb_diff = gold_train_new2['rougher.state.floatbank10_f_air'] - gold_train_new2['rougher.state.floatbank10_e_air']
r_state_fb["difference_f_e"] = r_state_fb_diff
r_e_nan = r_state_fb[r_state_fb['rougher.state.floatbank10_e_air'].isna()]

r_e_nan[r_e_nan['rougher.state.floatbank10_f_air'] < 844]
r_e_nan[(r_e_nan['rougher.state.floatbank10_f_air'] > 844) & (r_e_nan['rougher.state.floatbank10_f_air'] < 856)]

# Look at floatbank_f_air values between 844 - 856

r_state_fb_f = gold_train_new2[['rougher.state.floatbank10_a_air', 'rougher.state.floatbank10_b_air', 
                                'rougher.state.floatbank10_c_air', 'rougher.state.floatbank10_d_air','rougher.state.floatbank10_e_air',
                                'rougher.state.floatbank10_f_air']].copy()
r_state_fb_f = r_state_fb_f[(r_state_fb_f['rougher.state.floatbank10_f_air'] > 844) & 
    (r_state_fb_f['rougher.state.floatbank10_f_air'] < 856)]

r_state_fb_nan(r_state_fb_f)

r_state_fb

Difference Metrics (fb10_e_air & fb10_f_air)
-------------------------------------------
Length

1060

Median

0.030703981591784668

Mean

-25.26861356855608

Std

129.6028648493297

Min

-1072.1726457897248

# Look at floatbank_f_air values between 844 - 856

r_state_fb_e = gold_train_new2[['rougher.state.floatbank10_a_air', 'rougher.state.floatbank10_b_air', 
                                'rougher.state.floatbank10_c_air', 'rougher.state.floatbank10_d_air','rougher.state.floatbank10_e_air',
                                'rougher.state.floatbank10_f_air']].copy()
r_state_fb_e = r_state_fb_e[(r_state_fb_e['rougher.state.floatbank10_e_air'] > 844) & 
    (r_state_fb_f['rougher.state.floatbank10_e_air'] < 856)]

r_state_fb_nan(r_state_fb_e)

r_state_fb_e

Difference Metrics (fb10_e_air & fb10_f_air)
-------------------------------------------
Length

523

Median

0.06080069731820004

Mean

0.03194460157707011

Std

0.6767217288758983

Min

-4.909628087496003

# Remove the NaN values where fb_10_f_air is not between 844 - 856 (Training Set)
drop_rows = gold_train_new2[(gold_train_new2['rougher.state.floatbank10_f_air'] < 844) & (gold_train_new2['rougher.state.floatbank10_e_air'].isna())]
gold_train_new2 = gold_train_new2.drop(drop_rows.index)
gold_train_new2

# Replace the NaN values for fb_10_e_air with the values from fb_10_f_air (Training Set)

# Get the rows where fb10_f_air is between 844 - 856 and where 10_e_air is NaN
replace_values = gold_train_new2[(gold_train_new2['rougher.state.floatbank10_f_air'] > 844) & 
    (gold_train_new2['rougher.state.floatbank10_f_air'] < 856) & (gold_train_new2['rougher.state.floatbank10_e_air'].isna())]

# Separate the values from fb10_f_air
replace_values = replace_values['rougher.state.floatbank10_f_air']
replace_values

# Isolate the index
replace_values_index = replace_values.index

# Input fb10_f_air values where fb10_e_air values are NaN
gold_train_new2.loc[replace_values_index,['rougher.state.floatbank10_e_air']] = replace_values.values

# Remove the NaN values where fb_10_f_air is not between 844 - 856 (Full Set)

drop_rows_full = gold_full_new2[(gold_full_new2['rougher.state.floatbank10_f_air'] < 844) 
    & (gold_full_new2['rougher.state.floatbank10_e_air'].isna())]

gold_full_new2 = gold_full_new2.drop(drop_rows_full.index)
gold_full_new2

# The full dataset has the same number of NaN in fb10_e_air that fit the conditions
gold_full_new2[(gold_full_new2['rougher.state.floatbank10_f_air'] > 844) & 
    (gold_full_new2['rougher.state.floatbank10_f_air'] < 856) & (gold_full_new2['rougher.state.floatbank10_e_air'].isna())]

# Replace the NaN values for fb_10_e_air with the values from fb_10_f_air (Full Set)

# Get the rows where fb10_f_air is between 844 - 856 and where 10_e_air is NaN
replace_values_full = gold_full_new2[(gold_full_new2['rougher.state.floatbank10_f_air'] > 844) & 
    (gold_full_new2['rougher.state.floatbank10_f_air'] < 856) & (gold_full_new2['rougher.state.floatbank10_e_air'].isna())]

# Separate the values from fb10_f_air
replace_values_full = replace_values_full['rougher.state.floatbank10_f_air']
replace_values_full

# Isolate the index
replace_values_full_index = replace_values_full.index

# Input fb10_f_air values where fb10_e_air values are NaN
gold_full_new2.loc[replace_values_full_index,['rougher.state.floatbank10_e_air']] = replace_values_full.values

# Replace the NaN values for fb_10_e_air with the values from fb_10_f_air (Test Set)

# Get the rows where fb10_f_air is between 844 - 856 and where 10_e_air is NaN
replace_values_test = gold_test_new2[(gold_test_new2['rougher.state.floatbank10_f_air'] > 844) & 
    (gold_test_new2['rougher.state.floatbank10_f_air'] < 856) & (gold_test_new2['rougher.state.floatbank10_e_air'].isna())]

# Separate the values from fb10_f_air
replace_values_test = replace_values_test['rougher.state.floatbank10_f_air']
replace_values_test

# Isolate the index
replace_values_test_index = replace_values_test.index

# Input fb10_f_air values where fb10_e_air values are NaN
gold_test_new2.loc[replace_values_test_index,['rougher.state.floatbank10_e_air']] = replace_values_test.values

# Compare histograms

test_hist = gold_train_new2[['rougher.state.floatbank10_f_air','rougher.state.floatbank10_e_air']]

test_hist[test_hist.columns].hist(figsize = [10,4])
plt.show()

# View the missing data

gold_train_new2.isna().sum()

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                 259
primary_cleaner.input.depressant              189
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                190
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                       183
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate             257
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate             262
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      220
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# Compare Histograms for rougher.input.floatbank11_sulfate and rougher.input.floatbank10_sulfate

sulf_hist = gold_train_new2[['rougher.input.floatbank11_sulfate','rougher.input.floatbank10_sulfate']]

sulf_hist[sulf_hist.columns].hist(figsize = [10,6])

array([[<AxesSubplot:title={'center':'rougher.input.floatbank11_sulfate'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank10_sulfate'}>]],
      dtype=object)

# Understand the data

sulfate_train = sulf_hist.copy()
sulfate_train

len(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & (sulfate_train['rougher.input.floatbank11_sulfate'] < 5)])


sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5) & (sulfate_train['rougher.input.floatbank11_sulfate'] < 15)]


sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & (sulfate_train['rougher.input.floatbank11_sulfate'] < 0)]

sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 15) & (sulfate_train['rougher.input.floatbank11_sulfate'] < 25)]

sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & (sulfate_train['rougher.input.floatbank11_sulfate'] > 25 )]


sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'].isna()) & (sulfate_train['rougher.input.floatbank11_sulfate'] < 1)]

# Continue to understand the data

sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'].isna()) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] >= 0) & (sulfate_train['rougher.input.floatbank10_sulfate'] <= 1)]


sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'].notna()) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] >= 0) & (sulfate_train['rougher.input.floatbank10_sulfate'] <= 1)]


sulfate_train['sulfate_10_minus_11'] = sulfate_train['rougher.input.floatbank10_sulfate'] - sulfate_train['rougher.input.floatbank11_sulfate']

sulfate_train[sulfate_train['sulfate_10_minus_11'] < 1]

# When fb11_sulfate between 5.9 - 13.1; the difference tends to be extremely similar
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_10_minus_11'] > 1)]

# rougher.input.floatbank11_sulfate & rougher.input.floatbank10_sulfate comparison
sulfate_train = gold_train_new2.filter(like='_sulfate', axis = 1).copy()
sulfate_difference_train = sulfate_train['rougher.input.floatbank10_sulfate'] - sulfate_train['rougher.input.floatbank11_sulfate']
sulfate_train['sulfate_difference'] = sulfate_difference_train

print("DataFrame")
display(sulfate_train)
print()
print("Median")
display(sulfate_train.median())
print()
print("Mean")
display(sulfate_train.mean())
print()
print("Min")
display(sulfate_train.min())
print()
print("Max")
display(sulfate_train.max())
print()
print("Length")
display(len(sulfate_train))
print()
print()




print(f"Sulfate 11 between 5.9 - 13.1 & Differences Greater Than 1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 1)]))
print()
print()


print(f"Sulfate 11 between 5.9 - 13.1 & Difference Between 0 - 1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 0) 
    & (sulfate_train['sulfate_difference'] < 1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 0) 
    & (sulfate_train['sulfate_difference'] < 1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)]))
print()
print()

print(f"Sulfate 11 between 5.9 - 13.1 & Difference Between 0 - (-)1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < 0) 
    & (sulfate_train['sulfate_difference'] > -1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < 0) 
    & (sulfate_train['sulfate_difference'] > -1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)]))
print()
print()



print(f"Sulfate 11 between 5.9 - 13.1 & Differences Less Than -1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < -1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < -1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < -1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < -1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 5.9) & (sulfate_train['sulfate_difference'] < -1)]))
print()
print()


print(f"Sulfate 11 Less Than 1 & Differences Greater Than 1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 1)]))
print()
print()

print(f"Sulfate 11 Less Than 1 & Difference Between 10 - 15")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 10) & 
    (sulfate_train['sulfate_difference'] < 15)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 10) 
    & (sulfate_train['sulfate_difference'] < 15)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 10) & 
    (sulfate_train['sulfate_difference'] < 15)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 10) 
    & (sulfate_train['sulfate_difference'] < 15)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 10) & 
    (sulfate_train['sulfate_difference'] < 15)]))
print()
print()

print(f"Sulfate 11 Less Than 1 & Difference Between 0 - 1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 0) 
    & (sulfate_train['sulfate_difference'] < 1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 0) 
    & (sulfate_train['sulfate_difference'] < 1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] > 0) & 
    (sulfate_train['sulfate_difference'] < 1)]))
print()
print()

print(f"Sulfate 11 Less Than 1 & Difference Between 0 - (-)1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < 0) 
    & (sulfate_train['sulfate_difference'] > -1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < 0) 
    & (sulfate_train['sulfate_difference'] > -1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < 0) & 
    (sulfate_train['sulfate_difference'] > -1)]))
print()
print()



print(f"Sulfate 11 Less Than 1 & Differences Less Than -1")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < -1)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < -1)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < -1)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < -1)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 1) & (sulfate_train['sulfate_difference'] < -1)]))
print()
print()

DataFrame

Median

rougher.input.floatbank10_sulfate    11.708082
rougher.input.floatbank11_sulfate    11.413828
sulfate_difference                    0.000085
dtype: float64

Mean

rougher.input.floatbank10_sulfate    11.763041
rougher.input.floatbank11_sulfate    11.389276
sulfate_difference                    0.360227
dtype: float64

Min

rougher.input.floatbank10_sulfate     0.000044
rougher.input.floatbank11_sulfate     0.000049
sulfate_difference                  -12.977835
dtype: float64

Max

rougher.input.floatbank10_sulfate    36.118275
rougher.input.floatbank11_sulfate    37.980648
sulfate_difference                   23.746576
dtype: float64

Length

# Since 99.6 % of the data is ~0 when fb11_sulfate is between 5.9 - 13.1; 
# fill the fb10_sulfate NaN values, when fb11 is in the specified range, to fb11 values

# Training Set

training_values = gold_train_new2[(gold_train_new2['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (gold_train_new2['rougher.input.floatbank11_sulfate'] > 5.9) & (gold_train_new2['rougher.input.floatbank10_sulfate'].isna())]

training_values = training_values['rougher.input.floatbank11_sulfate']
training_values_index = training_values.index

gold_train_new2.loc[training_values_index, ['rougher.input.floatbank10_sulfate']] = training_values

gold_train_new2['rougher.input.floatbank10_sulfate'].isna().sum()

# Full DataSet

full_values = gold_full_new2[(gold_full_new2['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (gold_full_new2['rougher.input.floatbank11_sulfate'] > 5.9) & (gold_full_new2['rougher.input.floatbank10_sulfate'].isna())]

full_values = full_values['rougher.input.floatbank11_sulfate']
full_values_index = full_values.index

gold_full_new2.loc[full_values_index, ['rougher.input.floatbank10_sulfate']] = full_values

gold_full_new2['rougher.input.floatbank10_sulfate'].isna().sum()


# Test Set

test_values = gold_test_new2[(gold_test_new2['rougher.input.floatbank11_sulfate'] < 13.1) & 
    (gold_test_new2['rougher.input.floatbank11_sulfate'] > 5.9) & (gold_test_new2['rougher.input.floatbank10_sulfate'].isna())]

test_values = test_values['rougher.input.floatbank11_sulfate']
test_values_index = test_values.index

gold_test_new2.loc[test_values_index, ['rougher.input.floatbank10_sulfate']] = test_values

gold_test_new2['rougher.input.floatbank10_sulfate'].isna().sum()

254

# Drop the remaining missing values from fb10_sulfate (Training Set)
drop_train_values = gold_train_new2[gold_train_new2['rougher.input.floatbank10_sulfate'].isna()]

drop_train_values_index = drop_train_values.index

gold_train_new2 = gold_train_new2.drop(index = drop_train_values_index, axis = 1)


# Drop the remaining missing values from fb10_sulfate (Full Set)
drop_full_values = gold_full_new2[gold_full_new2['rougher.input.floatbank10_sulfate'].isna()]

drop_full_values_index = drop_full_values.index

gold_full_new2 = gold_full_new2.drop(index = drop_full_values_index, axis = 1)
gold_full_new2

# Compare Histograms (Training Set)
gold_train_new2[['rougher.input.floatbank10_sulfate','rougher.input.floatbank11_sulfate']].hist(figsize = [10,6])
plt.show()

# Look at the remaining NaN values
gold_train_new2.isna().sum()

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                  49
primary_cleaner.input.depressant               61
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                103
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                        42
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate               0
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate              62
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      220
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# Almost all are missing <1% data EXCEPT secondary_cleaner.state.floatbank2_a_air
# Drop the NaN values < 1%

# Training Set
drop_columns = ['rougher.input.floatbank11_sulfate','rougher.input.feed_rate','primary_cleaner.input.xanthate','primary_cleaner.input.depressant','primary_cleaner.input.sulfate']
gold_train_new2 = gold_train_new2.dropna(subset = drop_columns)
display(gold_train_new2)

# Full Set
gold_full_new2 = gold_full_new2.dropna(subset = drop_columns)
gold_full_new2

# Look into the last column `secondary_cleaner.state.floatbank2_a_air` with 1.5% missing data
gold_train_new2[gold_train_new2['secondary_cleaner.state.floatbank2_a_air'].isna()]

fb2 = gold_train_new2.filter(like = 'floatbank2', axis = 1).copy()
fb2 = fb2.filter(like = 'air', axis = 1)

fb2.hist(figsize = [10,5])
plt.show()

# Inspect the difference

fb2['a_b_difference'] = fb2['secondary_cleaner.state.floatbank2_a_air'] - fb2['secondary_cleaner.state.floatbank2_b_air']
fb2[(fb2['a_b_difference'] > -15) & (fb2['a_b_difference'] < 15)]

fb2[fb2['secondary_cleaner.state.floatbank2_b_air'] >= 35].max()

fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30) & (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35)].max()

fb2[(fb2['secondary_cleaner.state.floatbank2_a_air'].isna()) & fb2['secondary_cleaner.state.floatbank2_b_air']]

fb2[(fb2['secondary_cleaner.state.floatbank2_a_air'].isna())].max()

secondary_cleaner.state.floatbank2_a_air          NaN
secondary_cleaner.state.floatbank2_b_air    35.038455
a_b_difference                                    NaN
dtype: float64

fb2

# Only look at secondary_cleaner.state.floatbank2_b_air values between 22.9 & 35.1
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 22.9) & (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1)]

print("DataFrame")
display(fb2)
print()
print("Median")
display(fb2.median())
print()
print("Mean")
display(fb2.mean())
print()
print("Min")
display(fb2.min())
print()
print("Max")
display(fb2.max())
print()
print("Length")
display(len(fb2))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Differences Greater Than 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)]))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Difference Between 1 - 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 1) & 
    (fb2['a_b_difference'] <= 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 1) & 
    (fb2['a_b_difference'] <= 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 1) & 
    (fb2['a_b_difference'] <= 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 1) & 
    (fb2['a_b_difference'] <= 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 1) & 
    (fb2['a_b_difference'] <= 5)]))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Difference Between 0 - 1")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 1)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 1)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 1)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 1)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 1)]))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Difference Between 0 - (-)1")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 0) & 
    (fb2['a_b_difference'] >= -1)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 0) & 
    (fb2['a_b_difference'] >= -1)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 0) & 
    (fb2['a_b_difference'] >= -1)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 0) & 
    (fb2['a_b_difference'] >= -1)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 0) & 
    (fb2['a_b_difference'] >= -1)]))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Difference Between (-)1 - (-)5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -1) & 
    (fb2['a_b_difference'] >= -5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -1) & 
    (fb2['a_b_difference'] >= -5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -1) & 
    (fb2['a_b_difference'] >= -5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -1) & 
    (fb2['a_b_difference'] >= -5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -1) & 
    (fb2['a_b_difference'] >= -5)]))
print()
print()

print(f"fb_b between 22.9 - 35.1 & Differences Greater Than -5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < -5)]))
print()
print("HISTOGRAM: B(22.9 - 35.1)")
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 22.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= -15) & 
    (fb2['a_b_difference'] <= 25)].hist(figsize=[10,5])
plt.show()

DataFrame

Median

secondary_cleaner.state.floatbank2_a_air    30.023463
secondary_cleaner.state.floatbank2_b_air    27.021564
a_b_difference                               4.888594
dtype: float64

Mean

secondary_cleaner.state.floatbank2_a_air    29.654285
secondary_cleaner.state.floatbank2_b_air    25.029451
a_b_difference                               4.684958
dtype: float64

Min

secondary_cleaner.state.floatbank2_a_air     0.122013
secondary_cleaner.state.floatbank2_b_air     0.000000
a_b_difference                             -13.536875
dtype: float64

Max

secondary_cleaner.state.floatbank2_a_air    52.651399
secondary_cleaner.state.floatbank2_b_air    35.152122
a_b_difference                              24.654966
dtype: float64

Length

# Code used to find ranges within the main range (22.9 - 35.1)
with pd.option_context('display.max_columns',None):
    display(len(gold_train_new2[(gold_train_new2['secondary_cleaner.state.floatbank2_b_air'] >= 31.9) & 
        (gold_train_new2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & 
        (gold_train_new2['secondary_cleaner.state.floatbank2_a_air'].isna())]))

7

print("Grouped By fb_b in Increments")
print("-------------------------------------------")
print("B =  29.9 - 30.096: > 8")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] > 8)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] > 8)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] > 8)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] > 8)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] > 8)]))
print()
print()
print("B =  29.9 - 30.096: 2 - 8")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 2) & 
    (fb2['a_b_difference'] <= 8)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 2) & 
    (fb2['a_b_difference'] <= 8)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 2) & 
    (fb2['a_b_difference'] <= 8)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 2) & 
    (fb2['a_b_difference'] <= 8)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 2) & 
    (fb2['a_b_difference'] <= 8)]))
print()
print()
print("B =  29.9 - 30.096: < 2")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] < 2)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] < 2)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] < 2)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] < 2)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] < 2)]))
print()
print("HISTOGRAM: B(29.9 - 30.096)")
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 29.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.096) & (fb2['a_b_difference'] >= 0) & 
    (fb2['a_b_difference'] <= 15)].hist(figsize=[10,5])
plt.title("B(29.9 - 30.096)")
plt.show()

Grouped By fb_b in Increments
-------------------------------------------
B =  29.9 - 30.096: > 8
-----------------------------------
Median

secondary_cleaner.state.floatbank2_a_air    40.752771
secondary_cleaner.state.floatbank2_b_air    29.998159
a_b_difference                              10.773487
dtype: float64

Mean

secondary_cleaner.state.floatbank2_a_air    41.057590
secondary_cleaner.state.floatbank2_b_air    29.996569
a_b_difference                              11.061021
dtype: float64

Min

secondary_cleaner.state.floatbank2_a_air    38.028825
secondary_cleaner.state.floatbank2_b_air    29.937648
a_b_difference                               8.022474
dtype: float64

Max

secondary_cleaner.state.floatbank2_a_air    48.822781
secondary_cleaner.state.floatbank2_b_air    30.070166
a_b_difference                              18.830586
dtype: float64

Length

64

print("B =  30.8 - 30.9: > 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] > 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] > 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] > 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] > 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] > 5)]))
print()
print()
print("B =  30.8 - 30.9: 1 - 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)]))
print()
print()
print("B =  30.8 - 30.9: < 1")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] < 1)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] < 1)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] < 1)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] < 1)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] >= 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 30.9) & (fb2['a_b_difference'] < 1)]))
print()
print("HISTOGRAM: B(30.8 - 30.9)")
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 30.8) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 30.9) & (fb2['a_b_difference'] >= -50) & 
    (fb2['a_b_difference'] <= 50)].hist(figsize=[10,5])
plt.show()

B =  30.8 - 30.9: > 5
-----------------------------------
Median

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Mean

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Min

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Max

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Length

0

print("B =  31.9 - 32.1: > 4")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] > 4)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] > 4)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] > 4)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] > 4)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] > 4)]))
print()
print()
print("B =  31.9 - 32.1: 1 - 4")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 4)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 4)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 4)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 4)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 4)]))
print()
print()
print("B =  31.9 - 32.1: < 1")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] < 1)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] < 1)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] < 1)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] < 1)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] <= 32.1) & (fb2['a_b_difference'] < 1)]))
print()
print("HISTOGRAM: B(31.9 - 32.1)")
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 31.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 32.1) & (fb2['a_b_difference'] >= -2) & 
    (fb2['a_b_difference'] <= 5)].hist(figsize=[10,5])
plt.show()

B =  31.9 - 32.1: > 4
-----------------------------------
Median

secondary_cleaner.state.floatbank2_a_air    36.146185
secondary_cleaner.state.floatbank2_b_air    32.030408
a_b_difference                               4.115778
dtype: float64

Mean

secondary_cleaner.state.floatbank2_a_air    36.146185
secondary_cleaner.state.floatbank2_b_air    32.030408
a_b_difference                               4.115778
dtype: float64

Min

secondary_cleaner.state.floatbank2_a_air    36.146185
secondary_cleaner.state.floatbank2_b_air    32.030408
a_b_difference                               4.115778
dtype: float64

Max

secondary_cleaner.state.floatbank2_a_air    36.146185
secondary_cleaner.state.floatbank2_b_air    32.030408
a_b_difference                               4.115778
dtype: float64

Length

1

print("B =  34.9 - 35.1: > 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] > 5)]))
print()
print()
print("B =  34.9 - 35.1: 1 - 5")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= 1) & 
    (fb2['a_b_difference'] <= 5)]))
print()
print()
print("B =  34.9 - 35.1: < 1")
print("-----------------------------------")
print("Median")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 1)].median())
print()
print("Mean")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 1)].mean())
print()
print("Min")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 1)].min())
print()
print("Max")
display(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 1)].max())
print()
print("Length")
display(len(fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] < 1)]))
print()
print("HISTOGRAM: B(34.9 - 35.1)")
fb2[(fb2['secondary_cleaner.state.floatbank2_b_air'] > 34.9) & 
    (fb2['secondary_cleaner.state.floatbank2_b_air'] < 35.1) & (fb2['a_b_difference'] >= -2) & 
    (fb2['a_b_difference'] <= 1)].hist(figsize=[10,5])
plt.title("B(34.9 - 35.1)")
plt.show()

B =  34.9 - 35.1: > 5
-----------------------------------
Median

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Mean

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Min

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Max

secondary_cleaner.state.floatbank2_a_air   NaN
secondary_cleaner.state.floatbank2_b_air   NaN
a_b_difference                             NaN
dtype: float64

Length

0

gold_train_new2.isna().sum()

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                   0
primary_cleaner.input.depressant                0
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                  0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                         0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate               0
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate               0
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      217
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# Define a function to replace the specified values
def replace_values(df,start,stop, add):
    replace_a = df[(df['secondary_cleaner.state.floatbank2_b_air'] >= start) &
        (df['secondary_cleaner.state.floatbank2_b_air'] <= stop) &
        (df['secondary_cleaner.state.floatbank2_a_air'].isna())]
    
    replace_a_index = replace_a.index
    replace_a_values = replace_a['secondary_cleaner.state.floatbank2_b_air'] + add

    df.loc[replace_a_index,['secondary_cleaner.state.floatbank2_a_air']] = replace_a_values

display(gold_train_new2.isna().sum())
gold_full_new2.isna().sum()

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                   0
primary_cleaner.input.depressant                0
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                  0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                         0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate               0
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate               0
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      217
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                   0
primary_cleaner.input.depressant                0
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                  0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                         0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate               0
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate               0
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      220
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# When secondary_cleaner.state.floatbank2_b_air is between 29.9 - 30.096
# ~95.1% (1759) of secondary_cleaner.state.floatbank2_a_air is (~)+5.0
# The range for the gap was between 2 - 8 w/ roughly 1500 between a gap of 3-6

# Training dataset
gold_train_new2 = gold_train_new2.copy()
replace_values(gold_train_new2,29.9, 30.096, 5)

# Full dataset
gold_full_new2 = gold_full_new2.copy()
replace_values(gold_full_new2,29.9, 30.096, 5)

# Test dataset
gold_test_new2 = gold_test_new2.copy()
replace_values(gold_test_new2,29.9, 30.096, 5)

# When secondary_cleaner.state.floatbank2_b_air is between 30.8 - 30.9
# ~100% (7) of secondary_cleaner.state.floatbank2_a_air is (~)+1.2
# The range for the gap was between 1 - 2.2

# Training dataset
replace_values(gold_train_new2,30.8, 30.9, 1.2)

# Full dataset
replace_values(gold_full_new2,30.8, 30.9, 1.2)

# Test dataset
replace_values(gold_test_new2,30.8, 30.9, 1.2)

# When secondary_cleaner.state.floatbank2_b_air is between 31.9 - 32.1
# ~93.9% (509) of secondary_cleaner.state.floatbank2_a_air is (~)+3.0
# The range for the gap was between 1 - 4 

# Training dataset
replace_values(gold_train_new2,31.9, 32.1, 3)

# Full dataset
replace_values(gold_full_new2,31.9, 32.1, 3)

# Test dataset
replace_values(gold_test_new2,31.9, 32.1, 3)

# When secondary_cleaner.state.floatbank2_b_air is between 34.9 - 35.1
# ~100% (50) of secondary_cleaner.state.floatbank2_a_air is (~)-0.0
# The range for the gap was between < 1 with a min gap of -2.29 and a max gap of 0.39

# Training dataset
replace_values(gold_train_new2,34.9, 35.1, -0)

# Full dataset
replace_values(gold_full_new2,34.9, 35.1, -0)

# test dataset
replace_values(gold_test_new2,34.9, 35.1, -0)

gold_test_new2.isna().sum()

date                                            0
primary_cleaner.input.sulfate                 302
primary_cleaner.input.depressant              284
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                166
primary_cleaner.state.floatbank8_a_air         16
primary_cleaner.state.floatbank8_a_level       16
primary_cleaner.state.floatbank8_b_air         16
primary_cleaner.state.floatbank8_b_level       16
primary_cleaner.state.floatbank8_c_air         16
primary_cleaner.state.floatbank8_c_level       16
primary_cleaner.state.floatbank8_d_air         16
primary_cleaner.state.floatbank8_d_level       16
rougher.input.feed_ag                          16
rougher.input.feed_pb                          16
rougher.input.feed_rate                        40
rougher.input.feed_size                        22
rougher.input.feed_sol                         67
rougher.input.feed_au                          16
rougher.input.floatbank10_sulfate             254
rougher.input.floatbank10_xanthate            123
rougher.input.floatbank11_sulfate              55
rougher.input.floatbank11_xanthate            116
rougher.state.floatbank10_a_air                17
rougher.state.floatbank10_a_level              16
rougher.state.floatbank10_b_air                17
rougher.state.floatbank10_b_level              16
rougher.state.floatbank10_c_air                17
rougher.state.floatbank10_c_level              16
rougher.state.floatbank10_d_air                17
rougher.state.floatbank10_d_level              16
rougher.state.floatbank10_e_air                17
rougher.state.floatbank10_e_level              16
rougher.state.floatbank10_f_air                17
rougher.state.floatbank10_f_level              16
secondary_cleaner.state.floatbank2_a_air       20
secondary_cleaner.state.floatbank2_a_level     16
secondary_cleaner.state.floatbank2_b_air       23
secondary_cleaner.state.floatbank2_b_level     16
secondary_cleaner.state.floatbank3_a_air       34
secondary_cleaner.state.floatbank3_a_level     16
secondary_cleaner.state.floatbank3_b_air       16
secondary_cleaner.state.floatbank3_b_level     16
secondary_cleaner.state.floatbank4_a_air       16
secondary_cleaner.state.floatbank4_a_level     16
secondary_cleaner.state.floatbank4_b_air       16
secondary_cleaner.state.floatbank4_b_level     16
secondary_cleaner.state.floatbank5_a_air       16
secondary_cleaner.state.floatbank5_a_level     16
secondary_cleaner.state.floatbank5_b_air       16
secondary_cleaner.state.floatbank5_b_level     16
secondary_cleaner.state.floatbank6_a_air       16
secondary_cleaner.state.floatbank6_a_level     16
dtype: int64

# Remaining NaN values in secondary_cleaner.state.floatbank2_a_air 

display(gold_train_new2.isna().sum())
display(gold_full_new2.isna().sum())

date                                           0
final.output.recovery                          0
primary_cleaner.input.sulfate                  0
primary_cleaner.input.depressant               0
primary_cleaner.input.feed_size                0
primary_cleaner.input.xanthate                 0
primary_cleaner.state.floatbank8_a_air         0
primary_cleaner.state.floatbank8_a_level       0
primary_cleaner.state.floatbank8_b_air         0
primary_cleaner.state.floatbank8_b_level       0
primary_cleaner.state.floatbank8_c_air         0
primary_cleaner.state.floatbank8_c_level       0
primary_cleaner.state.floatbank8_d_air         0
primary_cleaner.state.floatbank8_d_level       0
rougher.input.feed_ag                          0
rougher.input.feed_pb                          0
rougher.input.feed_rate                        0
rougher.input.feed_size                        0
rougher.input.feed_sol                         0
rougher.input.feed_au                          0
rougher.input.floatbank10_sulfate              0
rougher.input.floatbank10_xanthate             0
rougher.input.floatbank11_sulfate              0
rougher.input.floatbank11_xanthate             0
rougher.state.floatbank10_a_air                0
rougher.state.floatbank10_a_level              0
rougher.state.floatbank10_b_air                0
rougher.state.floatbank10_b_level              0
rougher.state.floatbank10_c_air                0
rougher.state.floatbank10_c_level              0
rougher.state.floatbank10_d_air                0
rougher.state.floatbank10_d_level              0
rougher.state.floatbank10_e_air                0
rougher.state.floatbank10_e_level              0
rougher.state.floatbank10_f_air                0
rougher.state.floatbank10_f_level              0
secondary_cleaner.state.floatbank2_a_air      98
secondary_cleaner.state.floatbank2_a_level     0
secondary_cleaner.state.floatbank2_b_air       0
secondary_cleaner.state.floatbank2_b_level     0
secondary_cleaner.state.floatbank3_a_air       0
secondary_cleaner.state.floatbank3_a_level     0
secondary_cleaner.state.floatbank3_b_air       0
secondary_cleaner.state.floatbank3_b_level     0
secondary_cleaner.state.floatbank4_a_air       0
secondary_cleaner.state.floatbank4_a_level     0
secondary_cleaner.state.floatbank4_b_air       0
secondary_cleaner.state.floatbank4_b_level     0
secondary_cleaner.state.floatbank5_a_air       0
secondary_cleaner.state.floatbank5_a_level     0
secondary_cleaner.state.floatbank5_b_air       0
secondary_cleaner.state.floatbank5_b_level     0
secondary_cleaner.state.floatbank6_a_air       0
secondary_cleaner.state.floatbank6_a_level     0
dtype: int64

date                                            0
final.output.recovery                           0
primary_cleaner.input.sulfate                   0
primary_cleaner.input.depressant                0
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                  0
primary_cleaner.state.floatbank8_a_air          0
primary_cleaner.state.floatbank8_a_level        0
primary_cleaner.state.floatbank8_b_air          0
primary_cleaner.state.floatbank8_b_level        0
primary_cleaner.state.floatbank8_c_air          0
primary_cleaner.state.floatbank8_c_level        0
primary_cleaner.state.floatbank8_d_air          0
primary_cleaner.state.floatbank8_d_level        0
rougher.input.feed_ag                           0
rougher.input.feed_pb                           0
rougher.input.feed_rate                         0
rougher.input.feed_size                         0
rougher.input.feed_sol                          0
rougher.input.feed_au                           0
rougher.input.floatbank10_sulfate               0
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate               0
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                 0
rougher.state.floatbank10_a_level               0
rougher.state.floatbank10_b_air                 0
rougher.state.floatbank10_b_level               0
rougher.state.floatbank10_c_air                 0
rougher.state.floatbank10_c_level               0
rougher.state.floatbank10_d_air                 0
rougher.state.floatbank10_d_level               0
rougher.state.floatbank10_e_air                 0
rougher.state.floatbank10_e_level               0
rougher.state.floatbank10_f_air                 0
rougher.state.floatbank10_f_level               0
secondary_cleaner.state.floatbank2_a_air      101
secondary_cleaner.state.floatbank2_a_level      0
secondary_cleaner.state.floatbank2_b_air        0
secondary_cleaner.state.floatbank2_b_level      0
secondary_cleaner.state.floatbank3_a_air        0
secondary_cleaner.state.floatbank3_a_level      0
secondary_cleaner.state.floatbank3_b_air        0
secondary_cleaner.state.floatbank3_b_level      0
secondary_cleaner.state.floatbank4_a_air        0
secondary_cleaner.state.floatbank4_a_level      0
secondary_cleaner.state.floatbank4_b_air        0
secondary_cleaner.state.floatbank4_b_level      0
secondary_cleaner.state.floatbank5_a_air        0
secondary_cleaner.state.floatbank5_a_level      0
secondary_cleaner.state.floatbank5_b_air        0
secondary_cleaner.state.floatbank5_b_level      0
secondary_cleaner.state.floatbank6_a_air        0
secondary_cleaner.state.floatbank6_a_level      0
dtype: int64

# Drop the rest of the NaN values as they are under 1% (98/14,434) missing

# Training Set
gold_train_new2 = gold_train_new2.dropna()

# Full Dataset
gold_full_new2 = gold_train_new2.dropna()

display(gold_train_new2.isna().sum())
gold_full_new2.isna().sum()

date                                          0
final.output.recovery                         0
primary_cleaner.input.sulfate                 0
primary_cleaner.input.depressant              0
primary_cleaner.input.feed_size               0
primary_cleaner.input.xanthate                0
primary_cleaner.state.floatbank8_a_air        0
primary_cleaner.state.floatbank8_a_level      0
primary_cleaner.state.floatbank8_b_air        0
primary_cleaner.state.floatbank8_b_level      0
primary_cleaner.state.floatbank8_c_air        0
primary_cleaner.state.floatbank8_c_level      0
primary_cleaner.state.floatbank8_d_air        0
primary_cleaner.state.floatbank8_d_level      0
rougher.input.feed_ag                         0
rougher.input.feed_pb                         0
rougher.input.feed_rate                       0
rougher.input.feed_size                       0
rougher.input.feed_sol                        0
rougher.input.feed_au                         0
rougher.input.floatbank10_sulfate             0
rougher.input.floatbank10_xanthate            0
rougher.input.floatbank11_sulfate             0
rougher.input.floatbank11_xanthate            0
rougher.state.floatbank10_a_air               0
rougher.state.floatbank10_a_level             0
rougher.state.floatbank10_b_air               0
rougher.state.floatbank10_b_level             0
rougher.state.floatbank10_c_air               0
rougher.state.floatbank10_c_level             0
rougher.state.floatbank10_d_air               0
rougher.state.floatbank10_d_level             0
rougher.state.floatbank10_e_air               0
rougher.state.floatbank10_e_level             0
rougher.state.floatbank10_f_air               0
rougher.state.floatbank10_f_level             0
secondary_cleaner.state.floatbank2_a_air      0
secondary_cleaner.state.floatbank2_a_level    0
secondary_cleaner.state.floatbank2_b_air      0
secondary_cleaner.state.floatbank2_b_level    0
secondary_cleaner.state.floatbank3_a_air      0
secondary_cleaner.state.floatbank3_a_level    0
secondary_cleaner.state.floatbank3_b_air      0
secondary_cleaner.state.floatbank3_b_level    0
secondary_cleaner.state.floatbank4_a_air      0
secondary_cleaner.state.floatbank4_a_level    0
secondary_cleaner.state.floatbank4_b_air      0
secondary_cleaner.state.floatbank4_b_level    0
secondary_cleaner.state.floatbank5_a_air      0
secondary_cleaner.state.floatbank5_a_level    0
secondary_cleaner.state.floatbank5_b_air      0
secondary_cleaner.state.floatbank5_b_level    0
secondary_cleaner.state.floatbank6_a_air      0
secondary_cleaner.state.floatbank6_a_level    0
dtype: int64

date                                          0
final.output.recovery                         0
primary_cleaner.input.sulfate                 0
primary_cleaner.input.depressant              0
primary_cleaner.input.feed_size               0
primary_cleaner.input.xanthate                0
primary_cleaner.state.floatbank8_a_air        0
primary_cleaner.state.floatbank8_a_level      0
primary_cleaner.state.floatbank8_b_air        0
primary_cleaner.state.floatbank8_b_level      0
primary_cleaner.state.floatbank8_c_air        0
primary_cleaner.state.floatbank8_c_level      0
primary_cleaner.state.floatbank8_d_air        0
primary_cleaner.state.floatbank8_d_level      0
rougher.input.feed_ag                         0
rougher.input.feed_pb                         0
rougher.input.feed_rate                       0
rougher.input.feed_size                       0
rougher.input.feed_sol                        0
rougher.input.feed_au                         0
rougher.input.floatbank10_sulfate             0
rougher.input.floatbank10_xanthate            0
rougher.input.floatbank11_sulfate             0
rougher.input.floatbank11_xanthate            0
rougher.state.floatbank10_a_air               0
rougher.state.floatbank10_a_level             0
rougher.state.floatbank10_b_air               0
rougher.state.floatbank10_b_level             0
rougher.state.floatbank10_c_air               0
rougher.state.floatbank10_c_level             0
rougher.state.floatbank10_d_air               0
rougher.state.floatbank10_d_level             0
rougher.state.floatbank10_e_air               0
rougher.state.floatbank10_e_level             0
rougher.state.floatbank10_f_air               0
rougher.state.floatbank10_f_level             0
secondary_cleaner.state.floatbank2_a_air      0
secondary_cleaner.state.floatbank2_a_level    0
secondary_cleaner.state.floatbank2_b_air      0
secondary_cleaner.state.floatbank2_b_level    0
secondary_cleaner.state.floatbank3_a_air      0
secondary_cleaner.state.floatbank3_a_level    0
secondary_cleaner.state.floatbank3_b_air      0
secondary_cleaner.state.floatbank3_b_level    0
secondary_cleaner.state.floatbank4_a_air      0
secondary_cleaner.state.floatbank4_a_level    0
secondary_cleaner.state.floatbank4_b_air      0
secondary_cleaner.state.floatbank4_b_level    0
secondary_cleaner.state.floatbank5_a_air      0
secondary_cleaner.state.floatbank5_a_level    0
secondary_cleaner.state.floatbank5_b_air      0
secondary_cleaner.state.floatbank5_b_level    0
secondary_cleaner.state.floatbank6_a_air      0
secondary_cleaner.state.floatbank6_a_level    0
dtype: int64

# Test Sets Top Missing Values
def missing_pct(df):
    missing_values = df.isna().sum().values
    column_names = df.isna().sum().index
    return list((missing_values / 5856) * 100)

values = missing_pct(gold_test_new2)

values[:23]

[0.0,
 5.157103825136613,
 4.849726775956284,
 0.0,
 2.8346994535519126,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.273224043715847,
 0.6830601092896175,
 0.3756830601092896,
 1.1441256830601094,
 0.273224043715847,
 4.337431693989071,
 2.1004098360655736,
 0.9392076502732241,
 1.9808743169398908]

# Use Training data to impute Test Set

test_imp_columns = ['primary_cleaner.input.sulfate','primary_cleaner.input.depressant',
                    'rougher.input.floatbank10_sulfate','primary_cleaner.input.xanthate',
                    'rougher.input.floatbank10_xanthate','rougher.input.floatbank11_xanthate',
                    'rougher.input.feed_sol']

# Investigate the histograms from the Training dataset for the test_imp_columns
gold_train_new2[test_imp_columns].hist(figsize = [12,8])

array([[<AxesSubplot:title={'center':'primary_cleaner.input.sulfate'}>,
        <AxesSubplot:title={'center':'primary_cleaner.input.depressant'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank10_sulfate'}>],
       [<AxesSubplot:title={'center':'primary_cleaner.input.xanthate'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank10_xanthate'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank11_xanthate'}>],
       [<AxesSubplot:title={'center':'rougher.input.feed_sol'}>,
        <AxesSubplot:>, <AxesSubplot:>]], dtype=object)

with pd.option_context('display.max_columns',None):
    display(gold_train_new2)

# Define a function to replace df rows with the imputation
def replace_rows_median(df, df_imp, column):
    
    imputation = df_imp[column].median()
    values = df[column].fillna(imputation)
    index = values.index
    
    df.loc[index,[column]] = values

# replace primary_cleaner.input.sulfate with the median from the training set
replace_rows_median(gold_test_new2, gold_train_new2,'primary_cleaner.input.sulfate')

# Inspect the training set for remaining NaN values for the Test Set
gold_train_new2[test_imp_columns].hist(figsize = [12,8])

array([[<AxesSubplot:title={'center':'primary_cleaner.input.sulfate'}>,
        <AxesSubplot:title={'center':'primary_cleaner.input.depressant'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank10_sulfate'}>],
       [<AxesSubplot:title={'center':'primary_cleaner.input.xanthate'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank10_xanthate'}>,
        <AxesSubplot:title={'center':'rougher.input.floatbank11_xanthate'}>],
       [<AxesSubplot:title={'center':'rougher.input.feed_sol'}>,
        <AxesSubplot:>, <AxesSubplot:>]], dtype=object)

# Input the median for primary_cleaner.input.depressant and primary_cleaner.input.xanthate

replace_rows_median(gold_test_new2,gold_train_new2,'primary_cleaner.input.depressant')
replace_rows_median(gold_test_new2,gold_train_new2,'primary_cleaner.input.xanthate')

# Look at rougher.input.floatbank10_xanthate and rougher.input.floatbank11_xanthate for the test set

xan_test = gold_test_new2[['rougher.input.floatbank10_xanthate','rougher.input.floatbank11_xanthate']]
xan_test.hist()

xan_test_values = xan_test[(xan_test['rougher.input.floatbank10_xanthate'].isna()) & (xan_test['rougher.input.floatbank11_xanthate'].notna())]
xan_test11_values = xan_test_values['rougher.input.floatbank11_xanthate']
xan_test11_index = xan_test11_values.index
gold_test_new2.loc[xan_test11_index,['rougher.input.floatbank10_xanthate']] = xan_test11_values

# Input the median from the training set in the remaining values in fb10 and fb11_xanthate missing NaN values
replace_rows_median(gold_test_new2,gold_train_new2,'rougher.input.floatbank10_xanthate')
replace_rows_median(gold_test_new2,gold_train_new2,'rougher.input.floatbank11_xanthate')

gold_test_new2[test_imp_columns].hist()
gold_test_new2.isna().sum()

date                                            0
primary_cleaner.input.sulfate                   0
primary_cleaner.input.depressant                0
primary_cleaner.input.feed_size                 0
primary_cleaner.input.xanthate                  0
primary_cleaner.state.floatbank8_a_air         16
primary_cleaner.state.floatbank8_a_level       16
primary_cleaner.state.floatbank8_b_air         16
primary_cleaner.state.floatbank8_b_level       16
primary_cleaner.state.floatbank8_c_air         16
primary_cleaner.state.floatbank8_c_level       16
primary_cleaner.state.floatbank8_d_air         16
primary_cleaner.state.floatbank8_d_level       16
rougher.input.feed_ag                          16
rougher.input.feed_pb                          16
rougher.input.feed_rate                        40
rougher.input.feed_size                        22
rougher.input.feed_sol                         67
rougher.input.feed_au                          16
rougher.input.floatbank10_sulfate             254
rougher.input.floatbank10_xanthate              0
rougher.input.floatbank11_sulfate              55
rougher.input.floatbank11_xanthate              0
rougher.state.floatbank10_a_air                17
rougher.state.floatbank10_a_level              16
rougher.state.floatbank10_b_air                17
rougher.state.floatbank10_b_level              16
rougher.state.floatbank10_c_air                17
rougher.state.floatbank10_c_level              16
rougher.state.floatbank10_d_air                17
rougher.state.floatbank10_d_level              16
rougher.state.floatbank10_e_air                17
rougher.state.floatbank10_e_level              16
rougher.state.floatbank10_f_air                17
rougher.state.floatbank10_f_level              16
secondary_cleaner.state.floatbank2_a_air       20
secondary_cleaner.state.floatbank2_a_level     16
secondary_cleaner.state.floatbank2_b_air       23
secondary_cleaner.state.floatbank2_b_level     16
secondary_cleaner.state.floatbank3_a_air       34
secondary_cleaner.state.floatbank3_a_level     16
secondary_cleaner.state.floatbank3_b_air       16
secondary_cleaner.state.floatbank3_b_level     16
secondary_cleaner.state.floatbank4_a_air       16
secondary_cleaner.state.floatbank4_a_level     16
secondary_cleaner.state.floatbank4_b_air       16
secondary_cleaner.state.floatbank4_b_level     16
secondary_cleaner.state.floatbank5_a_air       16
secondary_cleaner.state.floatbank5_a_level     16
secondary_cleaner.state.floatbank5_b_air       16
secondary_cleaner.state.floatbank5_b_level     16
secondary_cleaner.state.floatbank6_a_air       16
secondary_cleaner.state.floatbank6_a_level     16
dtype: int64

# Fix fb_10_sulfate in a similar way to the training set; use sulfate 11 (2 - 5.9) 
# For the trainig set we used sulfate 11 (5.9 - 13.1)
# However, the majority of the training sets NaN values were not outside of this range

print(f"Sulfate 11 between 2 - 5.9 & Differences Greater Than 16")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)]))
print()
print()
print("Histogram: Sulfate 11 Between 2 - 5.9; Difference > 16")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] > 16)].hist()
plt.show()

print()
print()
print()
print(f"Sulfate 11 between 2 - 5.9 & Difference Between 6 - 16")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) 
    & (sulfate_train['sulfate_difference'] > 6)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) 
    & (sulfate_train['sulfate_difference'] > 6)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)]))
print()
print()
print("Histogram: Sulfate 11 Between 2 - 5.9; Difference Between 6 - 16")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].hist()
plt.show()


print()
print()
print()
print(f"Sulfate 11 between 2 - 5.9 & Differences Less Than 6")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)]))
print()
print()
print("Histogram: Sulfate 11 Between 2 - 5.9; Difference < 6")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 5.9) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 2) & (sulfate_train['sulfate_difference'] <= 6)].hist()
plt.show()

Sulfate 11 between 2 - 5.9 & Differences Greater Than 16
-----------------------------------
Median

rougher.input.floatbank10_sulfate    20.001514
rougher.input.floatbank11_sulfate     3.326154
sulfate_difference                   16.675361
dtype: float64

Mean

rougher.input.floatbank10_sulfate    20.001514
rougher.input.floatbank11_sulfate     3.326154
sulfate_difference                   16.675361
dtype: float64

Min

rougher.input.floatbank10_sulfate    20.001514
rougher.input.floatbank11_sulfate     3.326154
sulfate_difference                   16.675361
dtype: float64

Max

rougher.input.floatbank10_sulfate    20.001514
rougher.input.floatbank11_sulfate     3.326154
sulfate_difference                   16.675361
dtype: float64

Length

1

# Fix fb_10_sulfate in a similar way to the training set; use sulfate 11 (0 - 2) 


print(f"Sulfate 11 between 0 - 2 & Differences Greater Than 19")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)]))
print()
print()
print("Histogram: Sulfate 11 Between 0 - 2; Difference > 19")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 19)].hist()
plt.show()

print()
print()
print()
print(f"Sulfate 11 between 0 - 2 & Difference Between 5 - 19")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) & 
    (sulfate_train['sulfate_difference'] > 5)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) 
    & (sulfate_train['sulfate_difference'] > 5)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) & 
    (sulfate_train['sulfate_difference'] > 5)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) 
    & (sulfate_train['sulfate_difference'] > 5)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) & 
    (sulfate_train['sulfate_difference'] > 5)]))
print()
print()
print("Histogram: Sulfate 11 Between 0 - 2; Difference Between 5 - 19")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 19) & 
    (sulfate_train['sulfate_difference'] > 5)].hist()
plt.show()


print()
print()
print()
print(f"Sulfate 11 between 0 - 2 & Differences Less Than 5")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)]))
print()
print()
print("Histogram: Sulfate 11 Between 0 - 2; Difference < 5")
sulfate_train[(sulfate_train['rougher.input.floatbank11_sulfate'] <= 2) & 
    (sulfate_train['rougher.input.floatbank11_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 5)].hist()

Sulfate 11 between 0 - 2 & Differences Greater Than 19
-----------------------------------
Median

rougher.input.floatbank10_sulfate    21.996584
rougher.input.floatbank11_sulfate     0.013980
sulfate_difference                   21.980544
dtype: float64

Mean

rougher.input.floatbank10_sulfate    21.215349
rougher.input.floatbank11_sulfate     0.013487
sulfate_difference                   21.201862
dtype: float64

Min

rougher.input.floatbank10_sulfate    19.998256
rougher.input.floatbank11_sulfate     0.001877
sulfate_difference                   19.971806
dtype: float64

Max

rougher.input.floatbank10_sulfate    23.748453
rougher.input.floatbank11_sulfate     0.028495
sulfate_difference                   23.746576
dtype: float64

Length

24

# Define a function to replace the specified values
def replace_add(df, column_1, column_2, start,stop, add):
    replace = df[(df[column_2] >= start) &
        (df[column_2] <= stop) &
        (df[column_1].isna())]
    
    replace_index = replace.index
    replace_values = replace[column_2] + add

    df.loc[replace_index,[column_1]] = replace_values

# When Sulfate 11 Between 2 - 5.9 & Difference < 6
# Sulfate 10 ~ +0.0 difference ~98.2% of the time (280/285)
# column_1 = 10_sulfate
replace_add(gold_test_new2, 'rougher.input.floatbank10_sulfate','rougher.input.floatbank11_sulfate', 2, 5.9, 0.0)

# When Sulfate 11 Between 0 - 2 & Difference Between 5 - 19
# Sulfate 10 ~ +12.96 difference ~88.6% of the time (379/428)
# column_1 = 10_sulfate

replace_add(gold_test_new2, 'rougher.input.floatbank10_sulfate','rougher.input.floatbank11_sulfate', 0, 2, 12.96)

# Look into fb_11_sulfate


# Fix fb_11_sulfate in a similar way to the training set; use sulfate 10 (0 - 11) 
# For the trainig set we used sulfate 11 (5.9 - 13.1)
# However, the majority of the training sets NaN values were not outside of this range


print(f"Sulfate 10 between 0 - 11 & Differences Greater Than 16")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)]))
print()
print()
print("Histogram: Sulfate 10 Between 0 - 11; Difference > 16")
sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] > 16)].hist()
plt.show()

print()
print()
print()
print(f"Sulfate 10 between 0 - 11 & Difference Between 6 - 16")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) 
    & (sulfate_train['sulfate_difference'] > 6)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) 
    & (sulfate_train['sulfate_difference'] > 6)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)]))
print()
print()
print("Histogram: Sulfate 10 Between 0 - 11; Difference Between 6 - 16")
sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 16) & 
    (sulfate_train['sulfate_difference'] > 6)].hist()
plt.show()


print()
print()
print()
print(f"Sulfate 10 between 0 - 11 & Differences Less Than 6")
print("-----------------------------------")
print("Median")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)].median())
print()
print("Mean")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)].mean())
print()
print("Min")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)].min())
print()
print("Max")
display(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)].max())
print()
print("Length")
display(len(sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)]))
print()
print()
print("Histogram: Sulfate 10 Between 0 - 11; Difference < 6")
sulfate_train[(sulfate_train['rougher.input.floatbank10_sulfate'] <= 11) & 
    (sulfate_train['rougher.input.floatbank10_sulfate'] > 0) & (sulfate_train['sulfate_difference'] <= 6)].hist()
plt.show()

Sulfate 10 between 0 - 11 & Differences Greater Than 16
-----------------------------------
Median

rougher.input.floatbank10_sulfate   NaN
rougher.input.floatbank11_sulfate   NaN
sulfate_difference                  NaN
dtype: float64

Mean

rougher.input.floatbank10_sulfate   NaN
rougher.input.floatbank11_sulfate   NaN
sulfate_difference                  NaN
dtype: float64

Min

rougher.input.floatbank10_sulfate   NaN
rougher.input.floatbank11_sulfate   NaN
sulfate_difference                  NaN
dtype: float64

Max

rougher.input.floatbank10_sulfate   NaN
rougher.input.floatbank11_sulfate   NaN
sulfate_difference                  NaN
dtype: float64

Length

0

# When Sulfate 10 Between 0 - 11 & Difference < 6
# Sulfate 11 ~ -0.0 difference ~99.2% of the time (6076/6128)
# column_1 = 11_sulfate
replace_add(gold_test_new2, 'rougher.input.floatbank11_sulfate','rougher.input.floatbank10_sulfate', 0, 11, -0.0)

# rougher.input.feed_sol compared with rougher.input.feed_size
temp = gold_train_new2[['rougher.input.feed_size','rougher.input.feed_sol']].copy()
tempy = temp['rougher.input.feed_size'] - temp['rougher.input.feed_sol']
temp['size_sol_difference'] = tempy

temp.hist()

array([[<AxesSubplot:title={'center':'rougher.input.feed_size'}>,
        <AxesSubplot:title={'center':'rougher.input.feed_sol'}>],
       [<AxesSubplot:title={'center':'size_sol_difference'}>,
        <AxesSubplot:>]], dtype=object)

print(f"feed_size between 24 - 30 & Differences Greater Than 12")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)]))
print()
print()
print("Histogram: feed_size Between 24 - 30; Difference > 12")
temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] > 12)].hist()
plt.show()
print()
print()
print()
print(f"feed_size between 24 - 30 & Difference Between -5 - 12")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) & 
    (temp['size_sol_difference'] > -5)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) 
    & (temp['size_sol_difference'] > -5)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) & 
    (temp['size_sol_difference'] > -5)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) 
    & (temp['size_sol_difference'] > -5)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) & 
    (temp['size_sol_difference'] > -5)]))
print()
print()
print("Histogram: feedsize Between 24 - 30; Difference Between -5 - 12")
temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= 12) & 
    (temp['size_sol_difference'] > -5)].hist()
plt.show()
print()
print()
print()
print(f"feedsize between 24 - 30 & Differences Less Than -5")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)]))
print()
print()
print("Histogram: feed_size Between 24 - 30; Difference < -5")
temp[(temp['rougher.input.feed_size'] <= 30) & 
    (temp['rougher.input.feed_size'] > 24) & (temp['size_sol_difference'] <= -5)].hist()
plt.show()

feed_size between 24 - 30 & Differences Greater Than 12
-----------------------------------
Median

rougher.input.feed_size   NaN
rougher.input.feed_sol    NaN
size_sol_difference       NaN
dtype: float64

Mean

rougher.input.feed_size   NaN
rougher.input.feed_sol    NaN
size_sol_difference       NaN
dtype: float64

Min

rougher.input.feed_size   NaN
rougher.input.feed_sol    NaN
size_sol_difference       NaN
dtype: float64

Max

rougher.input.feed_size   NaN
rougher.input.feed_sol    NaN
size_sol_difference       NaN
dtype: float64

Length

0

# Input 25.2 in the test set for all values under 25.5 in feed_size
temp_new = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new_df = temp_new[(temp_new['rougher.input.feed_size'].notna()) & 
    (temp_new['rougher.input.feed_size'] < 25.5) & temp_new['rougher.input.feed_sol'].isna()]
temp_new_index = temp_new_df.index
temp_new_index
gold_test_new2.loc[temp_new_index,['rougher.input.feed_sol']] = 25.216914

# Input 39.273575 in the test set for values under 30 in feed_size when sol is NaN
temp_new39 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new39_df = temp_new39[(temp_new39['rougher.input.feed_size'].notna()) & 
    (temp_new39['rougher.input.feed_size'] < 30) & temp_new39['rougher.input.feed_sol'].isna()]
temp_new39_index = temp_new39_df.index
gold_test_new2.loc[temp_new39_index,['rougher.input.feed_sol']] = 39.273575

print(f"feed_size between 30 - 35 & Differences Greater Than 30")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)]))
print()
print()
print("Histogram: feed_size Between 30 - 35; Difference > 30")
temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] > 30)].hist()
plt.show()

print()
print()
print()
print(f"feed_size between 30 - 35 & Difference Between 4 - 30")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) & 
    (temp['size_sol_difference'] > 4)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) 
    & (temp['size_sol_difference'] > 4)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) & 
    (temp['size_sol_difference'] > 4)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) 
    & (temp['size_sol_difference'] > 4)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) & 
    (temp['size_sol_difference'] > 4)]))
print()
print()
print("Histogram: feedsize Between 30 - 35; Difference Between 4 - 30")
temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] <= 30) & 
    (temp['size_sol_difference'] > 4)].hist()
plt.show()
print()
print()
print()

print(f"feed_size between 30 - 35 & Differences Less Than 4")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)]))
print()
print()
print("Histogram: feed_size Between 30 - 35; Difference < 4")
temp[(temp['rougher.input.feed_size'] <= 35) & 
    (temp['rougher.input.feed_size'] > 30) & (temp['size_sol_difference'] < 4)].hist()
plt.show()

feed_size between 30 - 35 & Differences Greater Than 30
-----------------------------------
Median

rougher.input.feed_size    31.828881
rougher.input.feed_sol      0.295493
size_sol_difference        31.736166
dtype: float64

Mean

rougher.input.feed_size    32.595230
rougher.input.feed_sol      0.926313
size_sol_difference        31.668917
dtype: float64

Min

rougher.input.feed_size    31.747198
rougher.input.feed_sol      0.010000
size_sol_difference        31.451704
dtype: float64

Max

rougher.input.feed_size    34.209612
rougher.input.feed_sol      2.473446
size_sol_difference        31.818881
dtype: float64

Length

3

# Same instance as before, we will use the median for rougher.input.feed_sol - 36.606883
temp_new36 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new36_df = temp_new36[(temp_new36['rougher.input.feed_size'].notna()) & 
    (temp_new36['rougher.input.feed_size'] < 35) & temp_new36['rougher.input.feed_sol'].isna()]
temp_new36_index = temp_new36_df.index
gold_test_new2.loc[temp_new36_index,['rougher.input.feed_sol']] = 36.606883

print(f"feed_size between 35 - 38 & Differences Less Than 32")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 38) & 
    (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 38) & 
    (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 38) & 
    (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 38) & 
    (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 38) & 
    (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)]))
print()
print()
print("Histogram: feed_size Between 35 - 38; Difference < 32")
temp[(temp['rougher.input.feed_size'] <= 38) & (temp['rougher.input.feed_size'] > 35) & (temp['size_sol_difference'] < 32)].hist()
plt.show()

feed_size between 35 - 38 & Differences Less Than 32
-----------------------------------
Median

rougher.input.feed_size    37.057387
rougher.input.feed_sol     30.245821
size_sol_difference         6.905680
dtype: float64

Mean

rougher.input.feed_size    36.881165
rougher.input.feed_sol     27.823380
size_sol_difference         9.057785
dtype: float64

Min

rougher.input.feed_size    35.001415
rougher.input.feed_sol      5.495734
size_sol_difference        -3.183538
dtype: float64

Max

rougher.input.feed_size    37.997160
rougher.input.feed_sol     40.192872
size_sol_difference        31.367351
dtype: float64

Length

97

# Use the difference of 6.9 as the range is slightly closer than the sol median
replace_add(gold_test_new2,'rougher.input.feed_sol','rougher.input.feed_size',35,38,6.9)

print(f"feedsize between 40 - 50 & Differences Less Than 50")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)]))
print()
print()
print("Histogram: feed_size Between 40 - 50; Difference < 50")
temp[(temp['rougher.input.feed_size'] <= 50) & 
    (temp['rougher.input.feed_size'] > 40) & (temp['size_sol_difference'] <= 50)].hist()
plt.show()

feedsize between 40 - 50 & Differences Less Than 50
-----------------------------------
Median

rougher.input.feed_size    46.701653
rougher.input.feed_sol     34.544858
size_sol_difference        11.808177
dtype: float64

Mean

rougher.input.feed_size    46.346161
rougher.input.feed_sol     33.841244
size_sol_difference        12.504918
dtype: float64

Min

rougher.input.feed_size    40.000617
rougher.input.feed_sol      1.169889
size_sol_difference        -0.736386
dtype: float64

Max

rougher.input.feed_size    49.996193
rougher.input.feed_sol     45.955078
size_sol_difference        47.611883
dtype: float64

Length

3866

# A significant portion of this range (feed_size: 40 - 50) from the training set has a difference of ~11.8 from feed_sol
replace_add(gold_test_new2,'rougher.input.feed_sol','rougher.input.feed_size',40,50,11.8)

print(f"feed_size between 50 - 60 & Differences Less Than 55")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)]))
print()
print()
print("Histogram: feed_size Between 50 - 60; Difference < 55")
temp[(temp['rougher.input.feed_size'] <= 60) & 
    (temp['rougher.input.feed_size'] > 50) & (temp['size_sol_difference'] < 55)].hist()
plt.show()

feed_size between 50 - 60 & Differences Less Than 55
-----------------------------------
Median

rougher.input.feed_size    54.413676
rougher.input.feed_sol     36.802788
size_sol_difference        17.617468
dtype: float64

Mean

rougher.input.feed_size    54.547582
rougher.input.feed_sol     36.255374
size_sol_difference        18.292209
dtype: float64

Min

rougher.input.feed_size    50.004500
rougher.input.feed_sol      2.703490
size_sol_difference         6.438439
dtype: float64

Max

rougher.input.feed_size    59.998321
rougher.input.feed_sol     47.307449
size_sol_difference        53.629025
dtype: float64

Length

4927

# A large number in this range (feed_size: 50 - 60) have a median of 36.802788
# Input the median for this range
temp_new368 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new368_df = temp_new368[(temp_new368['rougher.input.feed_size'].notna()) & 
    (temp_new368['rougher.input.feed_size'] < 60) & temp_new368['rougher.input.feed_sol'].isna()]
temp_new368_index = temp_new368_df.index
gold_test_new2.loc[temp_new368_index,['rougher.input.feed_sol']] = 36.802788

print(f"feedsize between 60 - 70 & Differences Less Than 70")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)]))
print()
print("Histogram: feed_size Between 60 - 70; Difference < 70")
temp[(temp['rougher.input.feed_size'] <= 70) & 
    (temp['rougher.input.feed_size'] > 60) & (temp['size_sol_difference'] <= 70)].hist()
plt.show()

feedsize between 60 - 70 & Differences Less Than 70
-----------------------------------
Median

rougher.input.feed_size    64.135303
rougher.input.feed_sol     38.944012
size_sol_difference        25.295020
dtype: float64

Mean

rougher.input.feed_size    64.443953
rougher.input.feed_sol     38.459652
size_sol_difference        25.984302
dtype: float64

Min

rougher.input.feed_size    60.001225
rougher.input.feed_sol      0.010000
size_sol_difference        15.127535
dtype: float64

Max

rougher.input.feed_size    69.998638
rougher.input.feed_sol     48.363177
size_sol_difference        69.240783
dtype: float64

Length

2259

# A large number in this range (feed_size: 60 - 70) have a median of 38.944012
# Input the median for this range
temp_new389 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new389_df = temp_new389[(temp_new389['rougher.input.feed_size'].notna()) & 
    (temp_new389['rougher.input.feed_size'] < 70) & temp_new389['rougher.input.feed_sol'].isna()]
temp_new389_index = temp_new389_df.index
gold_test_new2.loc[temp_new389_index,['rougher.input.feed_sol']] = 38.944012

print(f"feed_size between 70 - 75 & Differences Greater Than 40")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)]))
print()
print()
print("Histogram: feed_size Between 70 - 75; Difference > 41")
temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] > 41)].hist()
plt.show()

print()
print()
print()

print(f"feed_size between 70 - 75 & Difference Between 28 - 41")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) & 
    (temp['size_sol_difference'] > 28)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) 
    & (temp['size_sol_difference'] > 28)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) & 
    (temp['size_sol_difference'] > 28)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) 
    & (temp['size_sol_difference'] > 28)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) & 
    (temp['size_sol_difference'] > 28)]))
print()
print()
print("Histogram: feedsize Between 70 - 75; Difference Between 28 - 41")
temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 41) & 
    (temp['size_sol_difference'] > 28)].hist()
plt.show()
print()
print()
print()


print(f"feedsize between 70 - 75 & Differences Less Than 28")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)]))
print()
print("Histogram: feed_size Between 70 - 75; Difference < 28")
temp[(temp['rougher.input.feed_size'] <= 75) & 
    (temp['rougher.input.feed_size'] > 70) & (temp['size_sol_difference'] <= 28)].hist()
plt.show()

feed_size between 70 - 75 & Differences Greater Than 40
-----------------------------------
Median

rougher.input.feed_size    73.703218
rougher.input.feed_sol     27.992614
size_sol_difference        43.988461
dtype: float64

Mean

rougher.input.feed_size    73.073895
rougher.input.feed_sol     27.353125
size_sol_difference        45.720770
dtype: float64

Min

rougher.input.feed_size    70.327801
rougher.input.feed_sol     16.194259
size_sol_difference        41.167205
dtype: float64

Max

rougher.input.feed_size    74.705631
rougher.input.feed_sol     33.315613
size_sol_difference        56.588363
dtype: float64

Length

31

# A large portion of this range (feed_size: 70 - 75) -  rougher.input.feed_sol median - 39.410044

temp_new394 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new394_df = temp_new394[(temp_new394['rougher.input.feed_size'].notna()) & 
    (temp_new394['rougher.input.feed_size'] < 75) & temp_new394['rougher.input.feed_sol'].isna()]
temp_new394_index = temp_new394_df.index
gold_test_new2.loc[temp_new394_index,['rougher.input.feed_sol']] = 39.410044

print(f"feedsize between 80 - 85 & Differences Greater Than 47")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] >= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)]))
print()
print("Histogram: feed_size Between 80 - 85; Difference > 47")
temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] > 47)].hist()
plt.show()
print()
print()
print()

print(f"feed_size between 80 - 85 & Difference Between 37 - 47")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) & 
    (temp['size_sol_difference'] > 37)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) 
    & (temp['size_sol_difference'] > 37)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) & 
    (temp['size_sol_difference'] > 37)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) 
    & (temp['size_sol_difference'] > 37)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) & 
    (temp['size_sol_difference'] > 37)]))
print()
print()
print("Histogram: feedsize Between 80 - 85; Difference Between 37 - 47")
temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 47) & 
    (temp['size_sol_difference'] > 37)].hist()
plt.show()
print()
print()
print()

print(f"feedsize between 80 - 85 & Differences Less Than 37")
print("-----------------------------------")
print("Median")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)].median())
print()
print("Mean")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)].mean())
print()
print("Min")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)].min())
print()
print("Max")
display(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)].max())
print()
print("Length")
display(len(temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)]))
print()
print("Histogram: feed_size Between 80 - 85; Difference < 37")
temp[(temp['rougher.input.feed_size'] <= 85) & 
    (temp['rougher.input.feed_size'] > 80) & (temp['size_sol_difference'] <= 37)].hist()
plt.show()

feedsize between 80 - 85 & Differences Greater Than 47
-----------------------------------
Median

rougher.input.feed_size    100.347223
rougher.input.feed_sol      35.775333
size_sol_difference         67.731138
dtype: float64

Mean

rougher.input.feed_size    82.682123
rougher.input.feed_sol     26.101812
size_sol_difference        56.580310
dtype: float64

Min

rougher.input.feed_size    80.058526
rougher.input.feed_sol      0.010000
size_sol_difference        47.074358
dtype: float64

Max

rougher.input.feed_size    84.864875
rougher.input.feed_sol     37.683318
size_sol_difference        81.133699
dtype: float64

Length

56

# A large portion of this range (feed_size: 80 - 85) -  rougher.input.feed_sol median - 40.657336

temp_new40 = gold_test_new2[['rougher.input.feed_size','rougher.input.feed_sol']]
temp_new40_df = temp_new40[(temp_new40['rougher.input.feed_size'].notna()) & 
    (temp_new40['rougher.input.feed_size'] < 83) & temp_new40['rougher.input.feed_sol'].isna()]
temp_new40_index = temp_new40_df.index
gold_test_new2.loc[temp_new40_index,['rougher.input.feed_sol']] = 40.657336

# Defina a function to input a spcecified columns median from the train dataset into the test dataset
def median_test_fillna(df_test,df_train, column):
    
    series = df_test[column]
    isna = series[series.isna()]
    series_index = isna.index
    train_median = df_train[column].median()
    
    df_test.loc[series_index,[column]] = train_median

# fill the rest of the NaN values for the test set


median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_c_air')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_c_level')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_d_air')
median_test_fillna(gold_test_new2, gold_train_new2,'primary_cleaner.state.floatbank8_d_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_ag')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_pb')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_rate')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_size')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_sol')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.feed_au')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.floatbank10_sulfate')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.input.floatbank11_sulfate')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_c_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_c_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_d_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_d_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_e_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_e_level')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_f_air')
median_test_fillna(gold_test_new2, gold_train_new2,'rougher.state.floatbank10_f_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank2_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank2_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank2_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank2_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank3_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank3_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank3_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank3_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank4_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank4_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank4_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank4_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank5_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank5_a_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank5_b_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank5_b_level')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank6_a_air')
median_test_fillna(gold_test_new2, gold_train_new2,'secondary_cleaner.state.floatbank6_a_level')

gold_test_new2.isna().sum()

date                                          0
primary_cleaner.input.sulfate                 0
primary_cleaner.input.depressant              0
primary_cleaner.input.feed_size               0
primary_cleaner.input.xanthate                0
primary_cleaner.state.floatbank8_a_air        0
primary_cleaner.state.floatbank8_a_level      0
primary_cleaner.state.floatbank8_b_air        0
primary_cleaner.state.floatbank8_b_level      0
primary_cleaner.state.floatbank8_c_air        0
primary_cleaner.state.floatbank8_c_level      0
primary_cleaner.state.floatbank8_d_air        0
primary_cleaner.state.floatbank8_d_level      0
rougher.input.feed_ag                         0
rougher.input.feed_pb                         0
rougher.input.feed_rate                       0
rougher.input.feed_size                       0
rougher.input.feed_sol                        0
rougher.input.feed_au                         0
rougher.input.floatbank10_sulfate             0
rougher.input.floatbank10_xanthate            0
rougher.input.floatbank11_sulfate             0
rougher.input.floatbank11_xanthate            0
rougher.state.floatbank10_a_air               0
rougher.state.floatbank10_a_level             0
rougher.state.floatbank10_b_air               0
rougher.state.floatbank10_b_level             0
rougher.state.floatbank10_c_air               0
rougher.state.floatbank10_c_level             0
rougher.state.floatbank10_d_air               0
rougher.state.floatbank10_d_level             0
rougher.state.floatbank10_e_air               0
rougher.state.floatbank10_e_level             0
rougher.state.floatbank10_f_air               0
rougher.state.floatbank10_f_level             0
secondary_cleaner.state.floatbank2_a_air      0
secondary_cleaner.state.floatbank2_a_level    0
secondary_cleaner.state.floatbank2_b_air      0
secondary_cleaner.state.floatbank2_b_level    0
secondary_cleaner.state.floatbank3_a_air      0
secondary_cleaner.state.floatbank3_a_level    0
secondary_cleaner.state.floatbank3_b_air      0
secondary_cleaner.state.floatbank3_b_level    0
secondary_cleaner.state.floatbank4_a_air      0
secondary_cleaner.state.floatbank4_a_level    0
secondary_cleaner.state.floatbank4_b_air      0
secondary_cleaner.state.floatbank4_b_level    0
secondary_cleaner.state.floatbank5_a_air      0
secondary_cleaner.state.floatbank5_a_level    0
secondary_cleaner.state.floatbank5_b_air      0
secondary_cleaner.state.floatbank5_b_level    0
secondary_cleaner.state.floatbank6_a_air      0
secondary_cleaner.state.floatbank6_a_level    0
dtype: int64

# Lastly, change the date to datetime
gold_train_new2['date'] = pd.to_datetime(gold_train_new2['date'])
gold_test_new2['date'] = pd.to_datetime(gold_test_new2['date'])
gold_full_new2['date'] = pd.to_datetime(gold_full_new2['date'])

# Take note of how the concentrations of metals (Au, Ag, Pb) change depending on the purification stage.

# Using original full dataset 

# Gold(Au)
au = gold_full.filter(like = 'au', axis = 1)
au_stages = ['rougher.input.feed_au',
    'rougher.output.concentrate_au',
    'rougher.output.tail_au',
    'primary_cleaner.output.concentrate_au',
    'primary_cleaner.output.tail_au',
    'secondary_cleaner.output.tail_au',
    'final.output.concentrate_au',
    'final.output.tail_au']
au_calculations = ['rougher.calculation.sulfate_to_au_concentrate',
       'rougher.calculation.floatbank10_sulfate_to_au_feed',
       'rougher.calculation.floatbank11_sulfate_to_au_feed',
       'rougher.calculation.au_pb_ratio']

# Silver(Ag)
ag = gold_full.filter(like = 'ag', axis = 1)
ag_stages = ['rougher.input.feed_ag',
             'rougher.output.concentrate_ag',
             'rougher.output.tail_ag',
             'primary_cleaner.output.concentrate_ag',
             'primary_cleaner.output.tail_ag',
             'secondary_cleaner.output.tail_ag',
             'final.output.concentrate_ag',
             'final.output.tail_ag']

# Lead(Pb)
pb = gold_full.filter(like = 'pb', axis = 1)
pb_stages = ['rougher.input.feed_pb', 
             'rougher.output.concentrate_pb',
             'rougher.output.tail_pb',
             'primary_cleaner.output.concentrate_pb',
             'primary_cleaner.output.tail_pb', 
             'secondary_cleaner.output.tail_pb',
             'final.output.concentrate_pb',
             'final.output.tail_pb']



# All Metals (Au, Ag, Pb)

all_metal_stages = au_stages + ag_stages + pb_stages

# Box plot - stages
au[au_stages].plot(kind='box', vert = False,figsize=(10, 5), title='Gold (Au) Concentration by Stage', grid = True)


# Line plot - Median Stages
au_median = au[au_stages].median()
stages = ['Rougher Feed','Rougher Conc','Rougher Tail',
          'Primary Cleaner Conc','Primary Cleaner Tail','Secondary Cleaner Tail','Final Conc','Final Tail']

plt.figure(figsize=(10,5))
plt.plot(stages, au_median, marker='o', label='Au median')
plt.title("Gold (Au) Median Concentration by Stage")
plt.ylabel("Concentration")
plt.xticks(rotation=45)
plt.grid(True)
plt.show()

# Show stats
au[au_stages].describe()

# Box plot - stages
ag[ag_stages].plot(kind = 'box', vert = False, figsize = (10,5),title = "Silver (Ag) Concentration by Stage", grid = True)

# Line plot - Median Stages
ag_median = ag[ag_stages].median()

plt.figure(figsize = (10,5))
plt.plot(stages,ag_median, marker='o', label = 'Ag median')
plt.title("Silver (Ag) Median Concentration by Stage")
plt.ylabel("Concentration")
plt.xticks(rotation = 45)
plt.grid(True)
plt.show()


# Show stats
ag[ag_stages].describe()

# Box plot - stages
pb[pb_stages].plot(kind='box', vert = False, figsize = (10,5), title = "Lead (Pb) Concentration by Stage", grid = True)

# Line plot - Median Stages
pb_median = pb[pb_stages].median()

plt.figure(figsize=(10,5))
plt.plot(stages, pb_median, marker = 'o', label = 'Pb median')
plt.title("Lead (Pb) Median Concentration by Stage")
plt.ylabel("Concentration")
plt.xticks(rotation = 45)
plt.grid(True)
plt.show()

# Show stats
pb[pb_stages].describe()

# Box plot - stages
gold_full[all_metal_stages].plot(kind='box', figsize = (12,8), vert = False, title = "All Metals (Au, Ag, Pb) Concentration by Stage", grid = True)

# Line plot - Median Stages


au_cols = ['Rougher Feed (Au)','Rougher Conc(Au)','Rougher Tail(Au)',
          'Primary Cleaner Conc(Au)','Primary Cleaner Tail(Au)','Secondary Cleaner Tail(Au)','Final Conc(Au)','Final Tail(Au)']

ag_cols = ['Rougher Feed (Ag)','Rougher Conc(Ag)','Rougher Tail(Ag)',
          'Primary Cleaner Conc(Ag)','Primary Cleaner Tail(Ag)','Secondary Cleaner Tail(Ag)','Final Conc(Ag)','Final Tail(Ag)']

pb_cols = ['Rougher Feed (Pb)','Rougher Conc(Pb)','Rougher Tail(Pb)',
          'Primary Cleaner Conc(Pb)','Primary Cleaner Tail(Pb)','Secondary Cleaner Tail(Pb)','Final Conc(Pb)','Final Tail(Pb)']


plt.figure(figsize=(12,8))
plt.plot(au_cols, au_median, marker = 'o', label = 'Au median')
plt.plot(ag_cols, ag_median, marker = 'o', label = 'Ag median')
plt.plot(pb_cols, pb_median, marker = 'o', label = 'Pb median')
plt.title("All Metals (Ag, Au, Pb) Concentration by Stage")
plt.ylabel('Concentration')
plt.xticks(rotation = 90)
plt.grid(True)
plt.legend()
plt.show()



# Show stats
with pd.option_context('display.max_columns', None):
    display(gold_full[all_metal_stages].describe())

all_imp = gold_full_new2[['rougher.input.feed_au','rougher.input.feed_ag','rougher.input.feed_pb']]

# Box plot - stages
all_imp.plot(kind='box', vert = False,figsize=(10, 5), 
             title='All Metals (Au, Ag, Pb) Concentration for Rougher Feed: Imputed Dataset', grid = True)

gold_full[['rougher.input.feed_au','rougher.input.feed_ag','rougher.input.feed_pb']].plot(
    kind='box', vert = False, figsize = (10,5), 
    title = "All Metals (Au, Ag, Pb) Concentration for Rougher Feed: Original Dataset", grid = True)

# Show stats
all_imp.describe()

# Compare the feed particle size distributions in the training set and in the test set. 
# If the distributions vary significantly, the model evaluation will be incorrect.

display(gold_train['rougher.input.feed_size'].describe())
display(gold_test['rougher.input.feed_size'].describe())

plt.figure(figsize=(10,5))
gold_train['rougher.input.feed_size'].plot(kind = 'kde', alpha = 0.7, label = 'Train_kde')
gold_test['rougher.input.feed_size'].plot(kind = 'kde', alpha = 0.7, label = 'Test_kde')
plt.legend()
plt.title("Feed Particle Size Distribution: Train vs Test (Original Dataset)")
plt.show()

plt.figure(figsize=(10,5))
gold_train_new2['rougher.input.feed_size'].plot(kind = 'kde', alpha = 0.7, label = 'Train_kde')
gold_test_new2['rougher.input.feed_size'].plot(kind = 'kde', alpha = 0.7, label = 'Test_kde')
plt.legend()
plt.title("Feed Particle Size Distribution: Train vs Test (Imputed Dataset)")
plt.show()

display(gold_train_new2['rougher.input.feed_size'].describe())
display(gold_test_new2['rougher.input.feed_size'].describe())

count    16443.000000
mean        58.676444
std         23.922591
min          9.659576
25%         47.575879
50%         54.104257
75%         65.051064
max        484.967466
Name: rougher.input.feed_size, dtype: float64

count    5834.000000
mean       55.937535
std        22.724254
min         0.046369
25%        43.890852
50%        50.002004
75%        61.638434
max       477.445473
Name: rougher.input.feed_size, dtype: float64

count    14336.000000
mean        59.916221
std         22.384221
min          9.659576
25%         48.873485
50%         55.253794
75%         65.994741
max        484.967466
Name: rougher.input.feed_size, dtype: float64

count    5856.000000
mean       55.934966
std        22.681559
min         0.046369
25%        43.898467
50%        50.109024
75%        61.608216
max       477.445473
Name: rougher.input.feed_size, dtype: float64

# Consider the total concentrations of all substances at different stages: 
# raw feed, rougher concentrate, and final concentrate. Do you notice any abnormal values in the total distribution? 
# If you do, is it worth removing such values from both samples? Describe the findings and eliminate anomalies.

conc = gold_full[['rougher.input.feed_au', 'rougher.output.concentrate_au', 'final.output.concentrate_au','rougher.input.feed_ag', 'rougher.output.concentrate_ag', 
                  'final.output.concentrate_ag','rougher.input.feed_pb', 'rougher.output.concentrate_pb', 'final.output.concentrate_pb']].copy()

conc['total_concentration_feed'] = conc['rougher.input.feed_au'] + conc['rougher.input.feed_ag'] + conc['rougher.input.feed_pb']
conc['total_concentration_rougher'] = conc['rougher.output.concentrate_au'] + conc['rougher.output.concentrate_ag'] + conc['rougher.output.concentrate_pb']
conc['total_concentration_final'] = conc['final.output.concentrate_au'] + conc['final.output.concentrate_ag'] + conc['final.output.concentrate_pb']

conc_imp = gold_full_new2[['rougher.input.feed_au','rougher.input.feed_ag','rougher.input.feed_pb']].copy()
conc_imp['total_concentration_feed'] = conc_imp['rougher.input.feed_au'] + conc_imp['rougher.input.feed_ag'] + conc_imp['rougher.input.feed_pb']

display(conc)
display(conc_imp)


display(conc['total_concentration_feed'].describe())
display(conc['total_concentration_rougher'].describe())
display(conc['total_concentration_final'].describe())
print()
print("Original Dataset")
print("-----------------")
conc[['total_concentration_feed','total_concentration_rougher','total_concentration_final']].hist(figsize = (10,5), label = "Total Concentration: Original Dataset")
plt.show()
print()
print()
print("Imputed Dataset (total_concentration_feed)")
print("-------------------------------------------")
conc_imp['total_concentration_feed'].hist(figsize=(10,5))
plt.show()
display(conc_imp['total_concentration_feed'].describe())

count    22471.000000
mean        18.985914
std          7.300593
min          0.000000
25%         16.553308
50%         19.629877
75%         23.618515
max         35.071987
Name: total_concentration_feed, dtype: float64

count    22618.000000
mean        35.654668
std         13.224242
min          0.000000
25%         37.382512
50%         39.979226
75%         42.192901
max         55.568687
Name: total_concentration_rougher, dtype: float64

count    22627.000000
mean        53.881912
std         17.697706
min          0.000000
25%         58.706155
50%         60.081820
75%         60.993449
max         65.575259
Name: total_concentration_final, dtype: float64

Original Dataset
-----------------


Imputed Dataset (total_concentration_feed)
-------------------------------------------

count    14336.000000
mean        20.291200
std          4.591283
min          0.030000
25%         16.999213
50%         19.437493
75%         23.087114
max         34.830220
Name: total_concentration_feed, dtype: float64

# Separate date column by year and month
gold_full_new3 = gold_full_new2.copy()
gold_full_new3['year'] = gold_full_new3['date'].dt.year
gold_full_new3['month'] = gold_full_new3['date'].dt.month


year_df = gold_full_new3[gold_full_new3['year'] == 2016]
year_df1 = gold_full_new3[gold_full_new3['year'] == 2017]
year_df2 = gold_full_new3[gold_full_new3['year'] == 2018]

year_df.groupby('month')['final.output.recovery'].mean().plot(kind='line', grid = True, label = '2016')
year_df1.groupby('month')['final.output.recovery'].mean().plot(kind='line', grid = True, label = '2017')
year_df2.groupby('month')['final.output.recovery'].mean().plot(kind='line', grid = True, label = '2018')
plt.title('Average Recovery by Month')
plt.ylabel('final.output.recovery')
plt.legend()
plt.show()

gold_full_new3 = gold_full_new3.drop(columns = ['date'])


# Do the same for the training set and the test set
gold_train_new3 = gold_train_new2.copy()
gold_train_new3['year'] = gold_train_new3['date'].dt.year
gold_train_new3['month'] = gold_train_new3['date'].dt.month
gold_train_new3 = gold_train_new3.drop(columns = ['date'])

gold_test_new3 = gold_test_new2.copy()
gold_test_new3['year'] = gold_test_new3['date'].dt.year
gold_test_new3['month'] = gold_test_new3['date'].dt.month
gold_test_new3 = gold_test_new3.drop(columns = ['date'])

# Create a temporary DF with rougher.output.recovery to do the sMAPE calculation
train_w_rougher = gold_train_new3.merge(
    gold_train[['rougher.output.recovery']],
    left_index=True,
    right_index=True,
    how='left'
)
train_w_rougher = train_w_rougher.dropna(subset=['rougher.output.recovery'])

train_w_rougher.isna().sum()

train_w_rougher

# Write a function to calculate the final sMAPE value.

def final_smape(df_train_final, final_column, df_train_rougher, rougher_column, df_test, 
                model_final = LinearRegression(), model_rougher = LinearRegression()):

    # Prepare features and targets
    features_train_final = df_train_final.drop([final_column], axis = 1)
    target_train_final = df_train_final[final_column]

    features_train_rougher = df_train_rougher.drop([rougher_column, final_column], axis = 1)
    target_train_rougher = df_train_rougher[rougher_column]
    
    # Fit models
    model_final.fit(features_train_final,target_train_final)
    model_rougher.fit(features_train_rougher,target_train_rougher)

    
    # Predict
    predicted_test_final = model_final.predict(df_test)
    predicted_test_rougher = model_rougher.predict(df_test)

    # Rename for formula
    y_true_final = target_train_final
    y_true_rougher = target_train_rougher

    y_pred_final = predicted_test_final
    y_pred_rougher = predicted_test_rougher

    # Smape Calculation
    smape_final = np.mean(((np.abs(y_true_final[:len(y_pred_final)] - y_pred_final)) / 
                           ((np.abs(y_true_final[:len(y_pred_final)]) + np.abs(y_pred_final)) / 2))) * 100
    smape_rougher = np.mean(((np.abs(y_true_rougher[:len(y_pred_rougher)] - y_pred_rougher)) / 
                             ((np.abs(y_true_rougher[:len(y_pred_rougher)]) + np.abs(y_pred_rougher)) / 2))) * 100


    # Final Smape
    return 0.25 * smape_rougher + 0.75 * smape_final

# Get the final_smape

# Use Linear Regression (default)
print("Linear Regression Model: Final sMAPE")
print("-------------------------------------")
display(final_smape(gold_train_new3,'final.output.recovery', train_w_rougher, 'rougher.output.recovery', 
            gold_test_new3, model_final = LinearRegression(), model_rougher = LinearRegression()))
print()
print()
# Use DecisionTreeRegressor
print("Decision Tree Model: Final sMAPE")
print("-------------------------------------")
display(final_smape(gold_train_new3,'final.output.recovery', train_w_rougher, 'rougher.output.recovery', 
            gold_test_new3, model_final = DecisionTreeRegressor(random_state=1234), model_rougher = DecisionTreeRegressor(random_state=12345)))
print()
print()
# Use RandomForestRegressor
print("Random Forest Model: Final sMAPE")
print("-------------------------------------")
display(final_smape(gold_train_new3,'final.output.recovery', train_w_rougher, 'rougher.output.recovery', 
            gold_test_new3, model_final = RandomForestRegressor(random_state=1234), model_rougher = RandomForestRegressor(random_state=1234)))

Linear Regression Model: Final sMAPE
-------------------------------------

14.63521422380902


Decision Tree Model: Final sMAPE
-------------------------------------

22.48113187928036


Random Forest Model: Final sMAPE
-------------------------------------

14.298332287669346

# Train different models. Evaluate them using cross-validation. Pick the best model and test it using the test sample.

# Define features and target
features_train_final = gold_train_new3.drop(['final.output.recovery'], axis=1)
target_train_final = gold_train_new3['final.output.recovery']

# Define models
model_rf = RandomForestRegressor(random_state=1234)
model_lr = LinearRegression()

# Perform 5-fold cross-validation
print("Random Forest Model: Cross-validating...")
scores_rf = cross_val_score(model_rf, features_train_final, target_train_final, cv=5, scoring='neg_mean_absolute_error')
print("Linear Regression Model: Cross-validating...")
scores_lr = cross_val_score(model_lr, features_train_final, target_train_final, cv=5, scoring='neg_mean_absolute_error')

# Convert from negative (since sklearn uses neg metrics for errors)
final_score_rf = -np.mean(scores_rf)
final_score_lr = -np.mean(scores_lr)


print('Average Cross-Validation Score (Random Forest):', final_score_rf)
print('Average Cross-Validation Score (Linear Regression):', final_score_lr)

Random Forest Model: Cross-validating...
Linear Regression Model: Cross-validating...
Average Cross-Validation Score (Random Forest): 6.262558587624503
Average Cross-Validation Score (Linear Regression): 6.8606775930853985

# Since rougher and final outputs were never in the test set, make DF's for the test set that align with the true targets
gold_full['date'] = pd.to_datetime(gold_full['date'])

# Create a test targets DF for sMAPE and predictions
test_dates = gold_test_new2['date']

test_w_targets = gold_full[gold_full['date'].isin(test_dates)][
    ['final.output.recovery', 'rougher.output.recovery']
].reset_index(drop=True)


# Merge targets with test features
gold_test_w_targets = pd.concat([gold_test_new3.reset_index(drop=True), test_w_targets], axis=1)

# Drop NaN rows for the missing targets
gold_test_w_true_targets_all = gold_test_w_targets.dropna()

# Get the true targets only
gold_test_w_true_targets_only = gold_test_w_true_targets_all[['final.output.recovery','rougher.output.recovery']]

# Get the true features only
gold_test_w_true_features = gold_test_w_true_targets_all.drop(columns = ['final.output.recovery','rougher.output.recovery'])

display(gold_test_w_true_features)
gold_test_new3

# Separate the targets from train_w_rougher; so both targets have the same number of predictions

true_train_targets = train_w_rougher[['final.output.recovery','rougher.output.recovery']]
true_train_targets_rougher = true_train_targets['rougher.output.recovery']
true_train_targets_rougher_full = train_w_rougher.drop(columns = ['final.output.recovery'])
true_train_targets_final = true_train_targets['final.output.recovery']
true_train_targets_final_full = train_w_rougher.drop(columns = ['rougher.output.recovery'])
true_train_features = train_w_rougher.drop(columns = ['final.output.recovery','rougher.output.recovery'])
true_train_targets_final_full

# Test the best model (RandomForest) and test it using the true test sample

# Get prediction for final output
best_model = model_rf  
best_model.fit(true_train_features, true_train_targets_final)

predictions_test_final = best_model.predict(gold_test_w_true_features)

print("Model Predictions for final.output.recovery (first 10):", predictions_test_final[:10])

Model Predictions for final.output.recovery (first 10): [67.81991271 68.35690678 68.14608863 67.68091849 69.04425219 68.95453817
 66.82514134 65.0654562  65.68501128 65.81068977]

# Get prediction for rougher output
best_model = model_rf  
best_model.fit(true_train_features, true_train_targets_rougher)

predictions_test_rougher = best_model.predict(gold_test_w_true_features)

print("Model Predictions for rougher.output.recovery (first 10):", predictions_test_rougher[:10])

Model Predictions for rougher.output.recovery (first 10): [89.32562713 85.24048458 86.04925568 84.58411688 88.13906292 86.41778105
 74.77973144 73.4266545  69.30563899 75.36474826]

# Call your original final_smape function for aligned features
print("Random Forest Model: Final sMAPE (with aligned features)")
print("-------------------------------------")
display(final_smape(true_train_targets_final_full,
                    'final.output.recovery',
                    train_w_rougher,
                    'rougher.output.recovery',
                    gold_test_w_true_features,
                    model_final=RandomForestRegressor(random_state = 1234),
                    model_rougher=RandomForestRegressor(random_state=1234)
))

Random Forest Model: Final sMAPE (with aligned features)
-------------------------------------

12.278200328481306

# Quick hypertuning for best results for final output
param_grid = {'max_depth': [3, 5, 7], 'n_estimators': [50, 100]}
grid = GridSearchCV(RandomForestRegressor(random_state=1234), param_grid, cv=5, scoring='neg_mean_absolute_error')
grid.fit(true_train_features, true_train_targets_final)

print("Best parameters (final output):", grid.best_params_)
print("Best score (final output):", -grid.best_score_)

Best parameters (final output): {'max_depth': 5, 'n_estimators': 100}
Best score (final output): 5.506685259260206

# Quick hypertuning for best results for rougher output
param_grid = {'max_depth': [3, 5, 7], 'n_estimators': [50, 100]}
grid = GridSearchCV(RandomForestRegressor(random_state=1234), param_grid, cv=5, scoring='neg_mean_absolute_error')
grid.fit(true_train_features, true_train_targets_rougher)

print("Best parameters (rougher output):", grid.best_params_)
print("Best score (rougher output):", -grid.best_score_)

Best parameters (rougher output): {'max_depth': 5, 'n_estimators': 100}
Best score (rougher output): 6.775727004776654

# Call your original final_smape function w/ hypertuned model
print("Random Forest Model: Final sMAPE (with hypertuned model)")
print("-------------------------------------")
display(final_smape(true_train_targets_final_full,
                    'final.output.recovery',
                    train_w_rougher,
                    'rougher.output.recovery',
                    gold_test_w_true_features,
                    model_final=RandomForestRegressor(random_state = 1234, max_depth = 5, n_estimators = 100),
                    model_rougher=RandomForestRegressor(random_state=1234, max_depth = 5, n_estimators = 100)
))

Random Forest Model: Final sMAPE (with hypertuned model)
-------------------------------------

11.477138030701516

	date	final.output.concentrate_ag	final.output.concentrate_pb	final.output.concentrate_sol	final.output.concentrate_au	final.output.recovery	final.output.tail_ag	final.output.tail_pb	final.output.tail_sol	final.output.tail_au	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
0	2016-01-15 00:00:00	6.055403	9.889648	5.507324	42.192020	70.541216	10.411962	0.895447	16.904297	2.143149	...	14.016835	-502.488007	12.099931	-504.715942	9.925633	-498.310211	8.079666	-500.470978	14.151341	-605.841980
1	2016-01-15 01:00:00	6.029369	9.968944	5.257781	42.701629	69.266198	10.462676	0.927452	16.634514	2.224930	...	13.992281	-505.503262	11.950531	-501.331529	10.039245	-500.169983	7.984757	-500.582168	13.998353	-599.787184
2	2016-01-15 02:00:00	6.055926	10.213995	5.383759	42.657501	68.116445	10.507046	0.953716	16.208849	2.257889	...	14.015015	-502.520901	11.912783	-501.133383	10.070913	-500.129135	8.013877	-500.517572	14.028663	-601.427363
3	2016-01-15 03:00:00	6.047977	9.977019	4.858634	42.689819	68.347543	10.422762	0.883763	16.532835	2.146849	...	14.036510	-500.857308	11.999550	-501.193686	9.970366	-499.201640	7.977324	-500.255908	14.005551	-599.996129
4	2016-01-15 04:00:00	6.148599	10.142511	4.939416	42.774141	66.927016	10.360302	0.792826	16.525686	2.055292	...	14.027298	-499.838632	11.953070	-501.053894	9.925709	-501.686727	7.894242	-500.356035	13.996647	-601.496691
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
16855	2018-08-18 06:59:59	3.224920	11.356233	6.803482	46.713954	73.755150	8.769645	3.141541	10.403181	1.529220	...	23.031497	-501.167942	20.007571	-499.740028	18.006038	-499.834374	13.001114	-500.155694	20.007840	-501.296428
16856	2018-08-18 07:59:59	3.195978	11.349355	6.862249	46.866780	69.049291	8.897321	3.130493	10.549470	1.612542	...	22.960095	-501.612783	20.035660	-500.251357	17.998535	-500.395178	12.954048	-499.895163	19.968498	-501.041608
16857	2018-08-18 08:59:59	3.109998	11.434366	6.886013	46.795691	67.002189	8.529606	2.911418	11.115147	1.596616	...	23.015718	-501.711599	19.951231	-499.857027	18.019543	-500.451156	13.023431	-499.914391	19.990885	-501.518452
16858	2018-08-18 09:59:59	3.367241	11.625587	6.799433	46.408188	65.523246	8.777171	2.819214	10.463847	1.602879	...	23.024963	-501.153409	20.054122	-500.314711	17.979515	-499.272871	12.992404	-499.976268	20.013986	-500.625471
16859	2018-08-18 10:59:59	3.598375	11.737832	6.717509	46.299438	70.281454	8.406690	2.517518	10.652193	1.389434	...	23.018622	-500.492702	20.020205	-500.220296	17.963512	-499.939490	12.990306	-500.080993	19.990336	-499.191575

	date	primary_cleaner.input.sulfate	primary_cleaner.input.depressant	primary_cleaner.input.feed_size	primary_cleaner.input.xanthate	primary_cleaner.state.floatbank8_a_air	primary_cleaner.state.floatbank8_a_level	primary_cleaner.state.floatbank8_b_air	primary_cleaner.state.floatbank8_b_level	primary_cleaner.state.floatbank8_c_air	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
0	2016-09-01 00:59:59	210.800909	14.993118	8.080000	1.005021	1398.981301	-500.225577	1399.144926	-499.919735	1400.102998	...	12.023554	-497.795834	8.016656	-501.289139	7.946562	-432.317850	4.872511	-500.037437	26.705889	-499.709414
1	2016-09-01 01:59:59	215.392455	14.987471	8.080000	0.990469	1398.777912	-500.057435	1398.055362	-499.778182	1396.151033	...	12.058140	-498.695773	8.130979	-499.634209	7.958270	-525.839648	4.878850	-500.162375	25.019940	-499.819438
2	2016-09-01 02:59:59	215.259946	12.884934	7.786667	0.996043	1398.493666	-500.868360	1398.860436	-499.764529	1398.075709	...	11.962366	-498.767484	8.096893	-500.827423	8.071056	-500.801673	4.905125	-499.828510	24.994862	-500.622559
3	2016-09-01 03:59:59	215.336236	12.006805	7.640000	0.863514	1399.618111	-498.863574	1397.440120	-499.211024	1400.129303	...	12.033091	-498.350935	8.074946	-499.474407	7.897085	-500.868509	4.931400	-499.963623	24.948919	-498.709987
4	2016-09-01 04:59:59	199.099327	10.682530	7.530000	0.805575	1401.268123	-500.808305	1398.128818	-499.504543	1402.172226	...	12.025367	-500.786497	8.054678	-500.397500	8.107890	-509.526725	4.957674	-500.360026	25.003331	-500.856333
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
5851	2017-12-31 19:59:59	173.957757	15.963399	8.070000	0.896701	1401.930554	-499.728848	1401.441445	-499.193423	1399.810313	...	13.995957	-500.157454	12.069155	-499.673279	7.977259	-499.516126	5.933319	-499.965973	8.987171	-499.755909
5852	2017-12-31 20:59:59	172.910270	16.002605	8.070000	0.896519	1447.075722	-494.716823	1448.851892	-465.963026	1443.890424	...	16.749781	-496.031539	13.365371	-499.122723	9.288553	-496.892967	7.372897	-499.942956	8.986832	-499.903761
5853	2017-12-31 21:59:59	171.135718	15.993669	8.070000	1.165996	1498.836182	-501.770403	1499.572353	-495.516347	1502.749213	...	19.994130	-499.791312	15.101425	-499.936252	10.989181	-498.347898	9.020944	-500.040448	8.982038	-497.789882
5854	2017-12-31 22:59:59	179.697158	15.438979	8.070000	1.501068	1498.466243	-500.483984	1497.986986	-519.200340	1496.569047	...	19.958760	-499.958750	15.026853	-499.723143	11.011607	-499.985046	9.009783	-499.937902	9.012660	-500.154284
5855	2017-12-31 23:59:59	181.556856	14.995850	8.070000	1.623454	1498.096303	-499.796922	1501.743791	-505.146931	1499.535978	...	20.034715	-500.728588	14.914199	-499.948518	10.986607	-500.658027	8.989497	-500.337588	8.988632	-500.764937

	date	final.output.concentrate_ag	final.output.concentrate_pb	final.output.concentrate_sol	final.output.concentrate_au	final.output.recovery	final.output.tail_ag	final.output.tail_pb	final.output.tail_sol	final.output.tail_au	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
0	2016-01-15 00:00:00	6.055403	9.889648	5.507324	42.192020	70.541216	10.411962	0.895447	16.904297	2.143149	...	14.016835	-502.488007	12.099931	-504.715942	9.925633	-498.310211	8.079666	-500.470978	14.151341	-605.841980
1	2016-01-15 01:00:00	6.029369	9.968944	5.257781	42.701629	69.266198	10.462676	0.927452	16.634514	2.224930	...	13.992281	-505.503262	11.950531	-501.331529	10.039245	-500.169983	7.984757	-500.582168	13.998353	-599.787184
2	2016-01-15 02:00:00	6.055926	10.213995	5.383759	42.657501	68.116445	10.507046	0.953716	16.208849	2.257889	...	14.015015	-502.520901	11.912783	-501.133383	10.070913	-500.129135	8.013877	-500.517572	14.028663	-601.427363
3	2016-01-15 03:00:00	6.047977	9.977019	4.858634	42.689819	68.347543	10.422762	0.883763	16.532835	2.146849	...	14.036510	-500.857308	11.999550	-501.193686	9.970366	-499.201640	7.977324	-500.255908	14.005551	-599.996129
4	2016-01-15 04:00:00	6.148599	10.142511	4.939416	42.774141	66.927016	10.360302	0.792826	16.525686	2.055292	...	14.027298	-499.838632	11.953070	-501.053894	9.925709	-501.686727	7.894242	-500.356035	13.996647	-601.496691
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
22711	2018-08-18 06:59:59	3.224920	11.356233	6.803482	46.713954	73.755150	8.769645	3.141541	10.403181	1.529220	...	23.031497	-501.167942	20.007571	-499.740028	18.006038	-499.834374	13.001114	-500.155694	20.007840	-501.296428
22712	2018-08-18 07:59:59	3.195978	11.349355	6.862249	46.866780	69.049291	8.897321	3.130493	10.549470	1.612542	...	22.960095	-501.612783	20.035660	-500.251357	17.998535	-500.395178	12.954048	-499.895163	19.968498	-501.041608
22713	2018-08-18 08:59:59	3.109998	11.434366	6.886013	46.795691	67.002189	8.529606	2.911418	11.115147	1.596616	...	23.015718	-501.711599	19.951231	-499.857027	18.019543	-500.451156	13.023431	-499.914391	19.990885	-501.518452
22714	2018-08-18 09:59:59	3.367241	11.625587	6.799433	46.408188	65.523246	8.777171	2.819214	10.463847	1.602879	...	23.024963	-501.153409	20.054122	-500.314711	17.979515	-499.272871	12.992404	-499.976268	20.013986	-500.625471
22715	2018-08-18 10:59:59	3.598375	11.737832	6.717509	46.299438	70.281454	8.406690	2.517518	10.652193	1.389434	...	23.018622	-500.492702	20.020205	-500.220296	17.963512	-499.939490	12.990306	-500.080993	19.990336	-499.191575

Metric	Value	Interpretation
Mean Absolute Error (MAE)	9.3e-15	~0% - Formula perfectly matches target values
Values that differ	0	All differences are floating-point rounding errors
Values differing by > 1e-20	0	Confirms perfect formula-target alignment

Category	Count	Percentage	Notes
Total training rows	16,860	100%	Complete dataset size
Valid formula results	14,287	84.7%	Rows where formula could be calculated
NaN in formula results	2,283	13.5%	Due to missing required input columns
NaN in target values	2,573	15.3%	Missing measurements in dataset
Rows after dropping NaNs	14,287	84.7%	Final comparison dataset

	predicted_values_train	actual_values_train
0	87.107763	87.107763
1	86.843261	86.843261
2	86.842308	86.842308
3	87.226430	87.226430
4	86.688794	86.688794
...	...	...
16855	89.574376	89.574376
16856	87.724007	87.724007
16857	88.890579	88.890579
16858	89.858126	89.858126
16859	89.514960	89.514960

Finding	Status	Impact
Formula Accuracy	✅ Perfect	Known formula perfectly reproduces target values
Data Coverage	⚠️ Good	84.7% of data usable for comparison
Missing Data Pattern	ℹ️ Expected	NaN values are typical in industrial datasets
Formula Reliability	✅ Excellent	Zero meaningful calculation errors detected

Parameter Type	Count	Reason for Exclusion
Output	28	Features only known after processing, not available at prediction time
Target	2	Used for stage-specific predictions
Calculations	4	Dependent on outputs/targets → potential data leakage

	miss_cnt_train	pct_missing_train
date	0	0.000000
final.output.concentrate_ag	72	0.427046
final.output.concentrate_pb	72	0.427046
final.output.concentrate_sol	370	2.194543
final.output.concentrate_au	71	0.421115
...	...	...
secondary_cleaner.state.floatbank5_a_level	85	0.504152
secondary_cleaner.state.floatbank5_b_air	85	0.504152
secondary_cleaner.state.floatbank5_b_level	84	0.498221
secondary_cleaner.state.floatbank6_a_air	103	0.610913
secondary_cleaner.state.floatbank6_a_level	85	0.504152

	miss_cnt	pct_missing	miss_cnt_train	pct_missing_train	miss_cnt_test	pct_missing_test
date	0	0.000	0	0.000	0.0	0.000
final.output.concentrate_ag	89	0.392	72	0.427	NaN	NaN
final.output.concentrate_pb	87	0.383	72	0.427	NaN	NaN
final.output.concentrate_sol	385	1.695	370	2.195	NaN	NaN
final.output.concentrate_au	86	0.379	71	0.421	NaN	NaN
final.output.recovery	1963	8.641	1521	9.021	NaN	NaN
final.output.tail_ag	83	0.365	66	0.391	NaN	NaN
final.output.tail_pb	200	0.880	183	1.085	NaN	NaN
final.output.tail_sol	271	1.193	145	0.860	NaN	NaN
final.output.tail_au	81	0.357	66	0.391	NaN	NaN
primary_cleaner.input.sulfate	1609	7.083	1307	7.752	302.0	5.175
primary_cleaner.input.depressant	1546	6.806	1262	7.485	284.0	4.866
primary_cleaner.input.feed_size	0	0.000	0	0.000	0.0	0.000
primary_cleaner.input.xanthate	1151	5.067	985	5.842	166.0	2.844
primary_cleaner.output.concentrate_ag	98	0.431	82	0.486	NaN	NaN
primary_cleaner.output.concentrate_pb	448	1.972	358	2.123	NaN	NaN
primary_cleaner.output.concentrate_sol	798	3.513	636	3.772	NaN	NaN
primary_cleaner.output.concentrate_au	98	0.431	82	0.486	NaN	NaN
primary_cleaner.output.tail_ag	102	0.449	83	0.492	NaN	NaN
primary_cleaner.output.tail_pb	122	0.537	99	0.587	NaN	NaN
primary_cleaner.output.tail_sol	351	1.545	281	1.667	NaN	NaN
primary_cleaner.output.tail_au	99	0.436	83	0.492	NaN	NaN
primary_cleaner.state.floatbank8_a_air	56	0.247	40	0.237	16.0	0.274
primary_cleaner.state.floatbank8_a_level	49	0.216	33	0.196	16.0	0.274
primary_cleaner.state.floatbank8_b_air	56	0.247	40	0.237	16.0	0.274
primary_cleaner.state.floatbank8_b_level	43	0.189	27	0.160	16.0	0.274
primary_cleaner.state.floatbank8_c_air	54	0.238	38	0.225	16.0	0.274
primary_cleaner.state.floatbank8_c_level	43	0.189	27	0.160	16.0	0.274
primary_cleaner.state.floatbank8_d_air	55	0.242	39	0.231	16.0	0.274
primary_cleaner.state.floatbank8_d_level	43	0.189	27	0.160	16.0	0.274
rougher.calculation.sulfate_to_au_concentrate	44	0.194	27	0.160	NaN	NaN
rougher.calculation.floatbank10_sulfate_to_au_feed	44	0.194	27	0.160	NaN	NaN
rougher.calculation.floatbank11_sulfate_to_au_feed	44	0.194	27	0.160	NaN	NaN
rougher.calculation.au_pb_ratio	1627	7.162	1242	7.367	NaN	NaN
rougher.input.feed_ag	98	0.431	82	0.486	16.0	0.274
rougher.input.feed_pb	244	1.074	228	1.352	16.0	0.274
rougher.input.feed_rate	553	2.434	513	3.043	40.0	0.685
rougher.input.feed_size	439	1.933	417	2.473	22.0	0.377
rougher.input.feed_sol	359	1.580	292	1.732	67.0	1.148
rougher.input.feed_au	99	0.436	83	0.492	16.0	0.274
rougher.input.floatbank10_sulfate	1301	5.727	1044	6.192	257.0	4.404
rougher.input.floatbank10_xanthate	469	2.065	346	2.052	123.0	2.108
rougher.input.floatbank11_sulfate	678	2.985	623	3.695	55.0	0.942
rougher.input.floatbank11_xanthate	2257	9.936	1904	11.293	353.0	6.049
rougher.output.concentrate_ag	98	0.431	82	0.486	NaN	NaN
rougher.output.concentrate_pb	98	0.431	82	0.486	NaN	NaN
rougher.output.concentrate_sol	190	0.836	162	0.961	NaN	NaN
rougher.output.concentrate_au	98	0.431	82	0.486	NaN	NaN
rougher.output.recovery	3119	13.730	2573	15.261	NaN	NaN
rougher.output.tail_ag	2737	12.049	2250	13.345	NaN	NaN
rougher.output.tail_pb	98	0.431	82	0.486	NaN	NaN
rougher.output.tail_sol	2736	12.044	2249	13.339	NaN	NaN
rougher.output.tail_au	2736	12.044	2249	13.339	NaN	NaN
rougher.state.floatbank10_a_air	70	0.308	53	0.314	17.0	0.291
rougher.state.floatbank10_a_level	69	0.304	53	0.314	16.0	0.274
rougher.state.floatbank10_b_air	70	0.308	53	0.314	17.0	0.291
rougher.state.floatbank10_b_level	69	0.304	53	0.314	16.0	0.274
rougher.state.floatbank10_c_air	70	0.308	53	0.314	17.0	0.291
rougher.state.floatbank10_c_level	62	0.273	46	0.273	16.0	0.274
rougher.state.floatbank10_d_air	75	0.330	58	0.344	17.0	0.291
rougher.state.floatbank10_d_level	67	0.295	51	0.302	16.0	0.274
rougher.state.floatbank10_e_air	620	2.729	603	3.577	17.0	0.291
rougher.state.floatbank10_e_level	67	0.295	51	0.302	16.0	0.274
rougher.state.floatbank10_f_air	75	0.330	58	0.344	17.0	0.291
rougher.state.floatbank10_f_level	74	0.326	58	0.344	16.0	0.274
secondary_cleaner.output.tail_ag	100	0.440	84	0.498	NaN	NaN
secondary_cleaner.output.tail_pb	116	0.511	96	0.569	NaN	NaN
secondary_cleaner.output.tail_sol	2215	9.751	1986	11.779	NaN	NaN
secondary_cleaner.output.tail_au	98	0.431	82	0.486	NaN	NaN
secondary_cleaner.state.floatbank2_a_air	383	1.686	363	2.153	20.0	0.343
secondary_cleaner.state.floatbank2_a_level	125	0.550	109	0.647	16.0	0.274
secondary_cleaner.state.floatbank2_b_air	178	0.784	155	0.919	23.0	0.394
secondary_cleaner.state.floatbank2_b_level	128	0.563	112	0.664	16.0	0.274
secondary_cleaner.state.floatbank3_a_air	131	0.577	97	0.575	34.0	0.583
secondary_cleaner.state.floatbank3_a_level	129	0.568	113	0.670	16.0	0.274
secondary_cleaner.state.floatbank3_b_air	124	0.546	108	0.641	16.0	0.274
secondary_cleaner.state.floatbank3_b_level	126	0.555	110	0.652	16.0	0.274
secondary_cleaner.state.floatbank4_a_air	145	0.638	129	0.765	16.0	0.274
secondary_cleaner.state.floatbank4_a_level	129	0.568	113	0.670	16.0	0.274
secondary_cleaner.state.floatbank4_b_air	108	0.475	92	0.546	16.0	0.274
secondary_cleaner.state.floatbank4_b_level	109	0.480	93	0.552	16.0	0.274
secondary_cleaner.state.floatbank5_a_air	101	0.445	85	0.504	16.0	0.274
secondary_cleaner.state.floatbank5_a_level	101	0.445	85	0.504	16.0	0.274
secondary_cleaner.state.floatbank5_b_air	101	0.445	85	0.504	16.0	0.274
secondary_cleaner.state.floatbank5_b_level	100	0.440	84	0.498	16.0	0.274
secondary_cleaner.state.floatbank6_a_air	119	0.524	103	0.611	16.0	0.274
secondary_cleaner.state.floatbank6_a_level	101	0.445	85	0.504	16.0	0.274

Category	Feature	Percentage
Full Only	final.output.tail_pb	0.880
Train Only	final.output.tail_sol	0.860
Test Only	rougher.input.feed_pb	0.274
Test Only	rougher.input.feed_rate	0.685
Test Only	rougher.input.feed_size	0.377
Test Only	rougher.input.floatbank11_sulfate	0.942
Test Only	rougher.state.floatbank10_e_air	0.291
Test Only	secondary_cleaner.state.floatbank2_a_air	0.343

Missing bins	All	Full	Train	Test	Full + Train	Full + Test
< 1%	40	1	1	6	21	–
1 – < 5%	2	1	1	3	10	–
5 – < 10%	1	1	–	–	5	1
10 – < 15%	–	1	2	–	3	–
≥ 15%	–	–	–	1	–	–
Total	43	4	5	9	39	1

Missing Data Level	Full Dataset	Training Dataset	Test Dataset
< 1%	62 features	62 features	45 features
1% - < 5%	14 features	14 features	5 features
5% - < 10%	7 features	5 features	2 features
10% - < 15%	4 features	5 features	0 features
≥ 15%	0 features	1 feature	0 features
Total with Missing Data	87 features	87 features	53 features

Missing Data Level	Full Dataset	Training Dataset	Test Dataset
< 1%	71.26%	71.26%	84.91%
1% - < 5%	16.10%	16.10%	9.43%
5% - < 10%	8.05%	5.75%	3.77%
10% - < 15%	4.60%	5.57%	0%
≥ 15%	0%	1.15%	0%
Total with Missing Data	87 features	87 features	53 features

	missing_cnt	missing_pct
secondary_cleaner.output.tail_sol	1778	11.591368
rougher.output.recovery	1190	7.758002
rougher.output.tail_ag	967	6.304192
rougher.output.tail_sol	966	6.297673
rougher.output.tail_au	966	6.297673
rougher.input.floatbank11_xanthate	779	5.078558
rougher.state.floatbank10_e_air	532	3.468283
primary_cleaner.output.concentrate_sol	408	2.659887
primary_cleaner.input.sulfate	381	2.483865
rougher.input.floatbank10_sulfate	375	2.444749
rougher.input.floatbank11_sulfate	357	2.327401
primary_cleaner.input.xanthate	276	1.799335
final.output.concentrate_sol	267	1.740661
primary_cleaner.input.depressant	257	1.675468
secondary_cleaner.state.floatbank2_a_air	230	1.499446
rougher.input.feed_rate	218	1.421214
primary_cleaner.output.concentrate_pb	161	1.049612

	predicted_values_train	actual_values_train
1	86.843261	86.843261
2	86.842308	86.842308
5	88.156912	88.156912
6	88.168065	88.168065
8	87.035862	87.035862
...	...	...
16849	91.675070	91.675070
16851	89.946627	89.946627
16854	91.816623	91.816623
16857	88.890579	88.890579
16859	89.514960	89.514960

	missing_cnt	missing_pct
secondary_cleaner.output.tail_sol	1952	9.405869
rougher.output.recovery	1314	6.331615
rougher.output.tail_ag	1065	5.131788
rougher.output.tail_sol	1064	5.126970
rougher.output.tail_au	1064	5.126970
rougher.input.floatbank11_xanthate	812	3.912687
rougher.state.floatbank10_e_air	532	2.563485
primary_cleaner.output.concentrate_sol	517	2.491206
primary_cleaner.input.sulfate	388	1.869609
rougher.input.floatbank10_sulfate	380	1.831061
rougher.input.floatbank11_sulfate	368	1.773238
primary_cleaner.input.xanthate	282	1.358840
final.output.concentrate_sol	267	1.286561
primary_cleaner.input.depressant	263	1.267287
secondary_cleaner.state.floatbank2_a_air	233	1.122729
rougher.input.feed_rate	221	1.064906

Feature	Missing Percentage
secondary_cleaner.output.tail_sol	~11.6%
rougher.output.recovery	~7.8%
rougher.output.tail_ag	~6.3%
rougher.output.tail_sol	~6.3%
rougher.output.tail_au	~6.3%
rougher.input.floatbank11_xanthate	~5.1%
rougher.state.floatbank10_e_air	~3.5%
primary_cleaner.output.concentrate_sol	~2.7%
primary_cleaner.input.sulfate	~2.5%
rougher.input.floatbank10_sulfate	~2.4%
rougher.input.floatbank11_sulfate	~2.3%
primary_cleaner.input.xanthate	~1.8%
final.output.concentrate_sol	~1.7%
primary_cleaner.input.depressant	~1.7%
secondary_cleaner.state.floatbank2_a_air	~1.5%
rougher.input.feed_rate	~1.4%
primary_cleaner.output.concentrate_pb	~1.0%

Feature	Missing Percentage
secondary_cleaner.output.tail_sol	~9.4%
rougher.output.recovery	~6.3%
rougher.output.tail_ag	~5.1%
rougher.output.tail_sol	~5.1%
rougher.output.tail_au	~5.1%
rougher.input.floatbank11_xanthate	~3.9%
rougher.state.floatbank10_e_air	~2.6%
primary_cleaner.output.concentrate_sol	~2.5%
primary_cleaner.input.sulfate	~1.9%
rougher.input.floatbank10_sulfate	~1.8%
rougher.input.floatbank11_sulfate	~1.8%
primary_cleaner.input.xanthate	~1.4%
final.output.concentrate_sol	~1.3%
primary_cleaner.input.depressant	~1.3%
secondary_cleaner.state.floatbank2_a_air	~1.1%
rougher.input.feed_rate	~1.1%

	date	final.output.recovery	primary_cleaner.input.sulfate	primary_cleaner.input.depressant	primary_cleaner.input.feed_size	primary_cleaner.input.xanthate	primary_cleaner.state.floatbank8_a_air	primary_cleaner.state.floatbank8_a_level	primary_cleaner.state.floatbank8_b_air	primary_cleaner.state.floatbank8_b_level	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
0	2016-01-15 00:00:00	70.541216	127.092003	10.128295	7.25	0.988759	1549.775757	-498.912140	1551.434204	-516.403442	...	14.016835	-502.488007	12.099931	-504.715942	9.925633	-498.310211	8.079666	-500.470978	14.151341	-605.841980
1	2016-01-15 01:00:00	69.266198	125.629232	10.296251	7.25	1.002663	1576.166671	-500.904965	1575.950626	-499.865889	...	13.992281	-505.503262	11.950531	-501.331529	10.039245	-500.169983	7.984757	-500.582168	13.998353	-599.787184
2	2016-01-15 02:00:00	68.116445	123.819808	11.316280	7.25	0.991265	1601.556163	-499.997791	1600.386685	-500.607762	...	14.015015	-502.520901	11.912783	-501.133383	10.070913	-500.129135	8.013877	-500.517572	14.028663	-601.427363
3	2016-01-15 03:00:00	68.347543	122.270188	11.322140	7.25	0.996739	1599.968720	-500.951778	1600.659236	-499.677094	...	14.036510	-500.857308	11.999550	-501.193686	9.970366	-499.201640	7.977324	-500.255908	14.005551	-599.996129
4	2016-01-15 04:00:00	66.927016	117.988169	11.913613	7.25	1.009869	1601.339707	-498.975456	1601.437854	-500.323246	...	14.027298	-499.838632	11.953070	-501.053894	9.925709	-501.686727	7.894242	-500.356035	13.996647	-601.496691
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
22711	2018-08-18 06:59:59	73.755150	123.381787	8.028927	6.50	1.304232	1648.421193	-400.382169	1648.742005	-400.359661	...	23.031497	-501.167942	20.007571	-499.740028	18.006038	-499.834374	13.001114	-500.155694	20.007840	-501.296428
22712	2018-08-18 07:59:59	69.049291	120.878188	7.962636	6.50	1.302419	1649.820162	-399.930973	1649.357538	-399.721222	...	22.960095	-501.612783	20.035660	-500.251357	17.998535	-500.395178	12.954048	-499.895163	19.968498	-501.041608
22713	2018-08-18 08:59:59	67.002189	105.666118	7.955111	6.50	1.315926	1649.166761	-399.888631	1649.196904	-399.677571	...	23.015718	-501.711599	19.951231	-499.857027	18.019543	-500.451156	13.023431	-499.914391	19.990885	-501.518452
22714	2018-08-18 09:59:59	65.523246	98.880538	7.984164	6.50	1.241969	1646.547763	-398.977083	1648.212240	-400.383265	...	23.024963	-501.153409	20.054122	-500.314711	17.979515	-499.272871	12.992404	-499.976268	20.013986	-500.625471
22715	2018-08-18 10:59:59	70.281454	95.248427	8.078957	6.50	1.283045	1648.759906	-399.862053	1650.135395	-399.957321	...	23.018622	-500.492702	20.020205	-500.220296	17.963512	-499.939490	12.990306	-500.080993	19.990336	-499.191575

Feature	Missing Count	Missing %
rougher.input.floatbank11_xanthate	779	5.08%
rougher.state.floatbank10_e_air	532	3.47%
primary_cleaner.input.sulfate	381	2.48%
rougher.input.floatbank10_sulfate	375	2.44%
rougher.input.floatbank11_sulfate	357	2.33%
primary_cleaner.input.xanthate	276	1.80%
primary_cleaner.input.depressant	257	1.68%
secondary_cleaner.state.floatbank2_a_air	230	1.50%
rougher.input.feed_rate	218	1.42%

Feature	Missing Count	Missing %
rougher.input.floatbank11_xanthate	812	3.91%
rougher.state.floatbank10_e_air	532	2.56%
primary_cleaner.input.sulfate	388	1.87%
rougher.input.floatbank10_sulfate	380	1.83%
rougher.input.floatbank11_sulfate	368	1.77%
primary_cleaner.input.xanthate	282	1.36%
primary_cleaner.input.depressant	263	1.27%
secondary_cleaner.state.floatbank2_a_air	233	1.12%
rougher.input.feed_rate	221	1.06%

	missing_cnt	missing_pct
rougher.input.feed_size	146	0.951822
rougher.input.feed_sol	143	0.932264
rougher.input.feed_pb	105	0.684530
rougher.input.floatbank10_xanthate	91	0.593259
secondary_cleaner.state.floatbank2_b_air	24	0.156464
rougher.state.floatbank10_c_air	16	0.104309
rougher.state.floatbank10_b_level	16	0.104309
rougher.state.floatbank10_b_air	16	0.104309
rougher.state.floatbank10_a_level	16	0.104309
rougher.state.floatbank10_a_air	16	0.104309
rougher.state.floatbank10_c_level	16	0.104309
rougher.state.floatbank10_f_level	15	0.097790
rougher.state.floatbank10_f_air	15	0.097790
rougher.state.floatbank10_e_level	15	0.097790
rougher.state.floatbank10_d_level	15	0.097790
rougher.state.floatbank10_d_air	15	0.097790
secondary_cleaner.state.floatbank4_a_air	9	0.058674
primary_cleaner.state.floatbank8_a_air	8	0.052155
primary_cleaner.state.floatbank8_b_air	8	0.052155
secondary_cleaner.state.floatbank3_a_air	7	0.045635
primary_cleaner.state.floatbank8_d_air	7	0.045635
primary_cleaner.state.floatbank8_c_air	6	0.039116
secondary_cleaner.state.floatbank6_a_air	3	0.019558
secondary_cleaner.state.floatbank5_b_level	1	0.006519
secondary_cleaner.state.floatbank5_a_air	1	0.006519
secondary_cleaner.state.floatbank4_a_level	1	0.006519
secondary_cleaner.state.floatbank5_b_air	1	0.006519
secondary_cleaner.state.floatbank3_b_air	1	0.006519
secondary_cleaner.state.floatbank4_b_air	1	0.006519
secondary_cleaner.state.floatbank5_a_level	1	0.006519
secondary_cleaner.state.floatbank3_b_level	1	0.006519
secondary_cleaner.state.floatbank4_b_level	1	0.006519
secondary_cleaner.state.floatbank6_a_level	1	0.006519
secondary_cleaner.state.floatbank3_a_level	1	0.006519
secondary_cleaner.state.floatbank2_b_level	1	0.006519
secondary_cleaner.state.floatbank2_a_level	1	0.006519
primary_cleaner.state.floatbank8_d_level	1	0.006519
primary_cleaner.state.floatbank8_c_level	1	0.006519
primary_cleaner.state.floatbank8_b_level	1	0.006519
primary_cleaner.state.floatbank8_a_level	1	0.006519
final.output.recovery	0	0.000000
rougher.input.feed_au	0	0.000000
rougher.input.feed_ag	0	0.000000
primary_cleaner.input.feed_size	0	0.000000
date	0	0.000000

	missing_cnt	missing_pct
rougher.input.feed_sol	175	0.843252
rougher.input.feed_size	147	0.708331
rougher.input.feed_pb	105	0.505951
rougher.input.floatbank10_xanthate	92	0.443309
secondary_cleaner.state.floatbank2_b_air	27	0.130102
secondary_cleaner.state.floatbank3_a_air	17	0.081916
rougher.state.floatbank10_b_air	16	0.077097
rougher.state.floatbank10_a_air	16	0.077097
rougher.state.floatbank10_c_air	16	0.077097
rougher.state.floatbank10_b_level	16	0.077097
rougher.state.floatbank10_a_level	16	0.077097
rougher.state.floatbank10_c_level	16	0.077097
rougher.state.floatbank10_d_level	15	0.072279
rougher.state.floatbank10_f_level	15	0.072279
rougher.state.floatbank10_f_air	15	0.072279
rougher.state.floatbank10_e_level	15	0.072279
rougher.state.floatbank10_d_air	15	0.072279
secondary_cleaner.state.floatbank4_a_air	9	0.043367
primary_cleaner.state.floatbank8_b_air	8	0.038549
primary_cleaner.state.floatbank8_a_air	8	0.038549
primary_cleaner.state.floatbank8_d_air	7	0.033730
primary_cleaner.state.floatbank8_c_air	6	0.028911
secondary_cleaner.state.floatbank6_a_air	3	0.014456
secondary_cleaner.state.floatbank3_b_level	1	0.004819
secondary_cleaner.state.floatbank4_a_level	1	0.004819
secondary_cleaner.state.floatbank5_b_level	1	0.004819
secondary_cleaner.state.floatbank5_a_level	1	0.004819
secondary_cleaner.state.floatbank5_a_air	1	0.004819
secondary_cleaner.state.floatbank4_b_level	1	0.004819
secondary_cleaner.state.floatbank3_b_air	1	0.004819
secondary_cleaner.state.floatbank4_b_air	1	0.004819
secondary_cleaner.state.floatbank5_b_air	1	0.004819
secondary_cleaner.state.floatbank6_a_level	1	0.004819
secondary_cleaner.state.floatbank3_a_level	1	0.004819
secondary_cleaner.state.floatbank2_b_level	1	0.004819
secondary_cleaner.state.floatbank2_a_level	1	0.004819
primary_cleaner.state.floatbank8_d_level	1	0.004819
primary_cleaner.state.floatbank8_c_level	1	0.004819
primary_cleaner.state.floatbank8_b_level	1	0.004819
primary_cleaner.state.floatbank8_a_level	1	0.004819
final.output.recovery	0	0.000000
rougher.input.feed_au	0	0.000000
rougher.input.feed_ag	0	0.000000
primary_cleaner.input.feed_size	0	0.000000
date	0	0.000000

Feature	Missing Count	Missing %
rougher.input.floatbank11_xanthate	654	4.40%
rougher.state.floatbank10_e_air	508	3.42%
rougher.input.floatbank11_sulfate	262	1.76%
primary_cleaner.input.sulfate	259	1.74%
rougher.input.floatbank10_sulfate	259	1.74%
secondary_cleaner.state.floatbank2_a_air	220	1.48%
primary_cleaner.input.xanthate	190	1.28%
primary_cleaner.input.depressant	189	1.27%
rougher.input.feed_rate	183	1.23%

Feature	Missing Count	Missing %
rougher.input.floatbank11_xanthate	684	3.38%
rougher.state.floatbank10_e_air	508	2.51%
rougher.input.floatbank11_sulfate	272	1.34%
primary_cleaner.input.sulfate	263	1.30%
rougher.input.floatbank10_sulfate	261	1.29%
secondary_cleaner.state.floatbank2_a_air	223	1.10%
primary_cleaner.input.xanthate	193	0.95%
primary_cleaner.input.depressant	193	0.95%
rougher.input.feed_rate	185	0.91%

Project Overview¶

Technical Understanding¶

Environment Setup & Required Libraries¶

Download & Prepare the Data¶

Recovery & MAE Calculations¶

Examine Unavailable Features¶

Process Data¶

Fix Missing Values¶

Drop Columns that May Cause Leakage¶

Missing Values¶

Missing Values - < 1% Missing Data¶

Missing Values - Imputation¶

Missing Values - Impute Test Set¶

Analyze the Data¶

Concentrations of Metals Accross Purification Stages (Au, Ag, Pb): Original Dataset¶

Gold(Au): Original Dataset¶

Silver(Ag): Original Dataset¶

Lead(Pb): Original Dataset¶

Compare (Au, Ag, Pb): Original Dataset¶

Imputed Dataset¶

Compare feed_size: Train vs. Test¶

Total Concentration Sanity Check¶

Build the Model¶

symmetric Mean Absolute Percentage Error¶

Evaluating Models Using Cross-Validation¶

Final Conclusion¶

Data Preprocessing Summary¶

Outlier Analysis: Original vs. Imputed Data¶

Distribution Consistency: Train-Test Alignment¶

Model Performance Evolution¶

Key Technical Achievements¶

Business Impact¶

Recommendations for Deployment¶

Condensed Conculsions per Section¶

MAE Calculations for Rougher Recovery Calculation (Training Set)¶

Missing Test Features¶

Missing Values Overview¶

Missing Data Summary (Target NaN's Removed) - Side-by-Side Comparison¶

Missing Data Analysis - Cleaned Datasets¶

Missing Data Analysis - Updated Cleaned Datasets¶

Xanthate Difference Analysis: rougher.input.floatbank##_xanthate¶

Floatbank Air Imputation Strategy¶

Sulfate Imputation Strategy Conclusion¶

Feed Sol Imputation Strategy Conclusion¶

Remaining Columns Imputation Strategy (<1% Missing Data)¶

Original Dataset: Outlier and Distribution Summary¶

Original Dataset Vs. Imputed Dataset: Outlier Analysis and Distribution Summary¶

Feed_Size Distribution: Train Vs. Test¶

Total Concentration Comparison: Original vs Imputed¶

sMAPE Analysis¶

Cross-Validation Analysis¶

Final Model Evaluation: Random Forest on True Test Set¶

Supplemental Information¶

MAE Calculation & Rougher Recovery (Training Set)¶

Features NOT in the Test Set (34 Total):¶

Missing Values Overview (≥ 1%)¶

Missing Data Summary - Side-by-Side Comparison (Target NaN's removed)¶

Missing Data Analysis - Cleaned Datasets¶

Missing Data Analysis - Updated Cleaned Datasets¶

Xanthate Difference Analysis: rougher.input.floatbank##_xanthate¶

Floatbank Air Difference Analysis (fb10_e_air & fb10_f_air) - Detailed Breakdown**¶

Sulfate Difference Analysis: rougher.input.floatbank10_sulfate vs floatbank11_sulfate¶

Imputation Strategy for rougher.input.floatbank10_sulfate: for Test Set¶

Imputation Strategy for rougher.input.floatbank11_sulfate: for Test Set¶

Imputation Strategy for rougher.input.feed_sol: for Test Set¶

Imputation Strategy for rougher.input.feed_sol: for Test Set¶

Imputation Strategy for Remaining Columns (<1% Missing Data: Test Set)¶

Original Dataset: Outlier Analysis and Distribution Assessment¶

Original Dataset Vs. Imputed Dataset: Outlier Analysis and Distribution Assessment¶

Feed_Size Distribution: Train Vs. Test¶

Total Concentration Analysis¶

sMAPE Analysis¶

Cross-Validation Analysis¶

Final Model Evaluation: Random Forest on True Test Set¶

Imputation Strategy for `rougher.input.floatbank10_sulfate`: for Test Set¶

Imputation Strategy for `rougher.input.floatbank11_sulfate`: for Test Set¶

Imputation Strategy for `rougher.input.feed_sol`: for Test Set¶

Imputation Strategy for `rougher.input.feed_sol`: for Test Set¶

	rougher.input.floatbank11_xanthate	rougher.state.floatbank10_e_air	rougher.input.floatbank11_sulfate	primary_cleaner.input.sulfate	rougher.input.floatbank10_sulfate	secondary_cleaner.state.floatbank2_a_air	primary_cleaner.input.xanthate	primary_cleaner.input.depressant	rougher.input.feed_rate
count	14201.000000	14347.000000	14593.000000	14596.000000	14596.000000	14635.000000	14665.000000	14666.000000	14672.000000
mean	5.888175	1078.990959	11.389359	134.643263	11.763041	29.341060	0.879605	8.929893	467.518312
std	1.149914	202.268601	3.734254	39.951815	3.313112	6.334380	0.386629	3.437355	111.056786
min	0.000290	-1.970275	0.000049	0.003112	0.000044	0.077503	0.000006	0.000000	0.001166
25%	5.198196	951.610899	9.506037	107.939017	9.855929	25.094668	0.601735	6.051069	408.014630
50%	5.999159	1049.988150	11.414551	134.315614	11.708082	30.017078	0.888980	8.043323	498.403275
75%	6.700692	1199.872051	13.500338	161.428026	13.684285	34.878417	1.101912	11.912753	545.801298
max	9.698407	1922.636637	37.980648	251.999948	36.118275	60.000000	2.512968	20.673152	717.508837

	rougher.input.floatbank11_xanthate	rougher.state.floatbank10_e_air	rougher.input.floatbank11_sulfate	primary_cleaner.input.sulfate	rougher.input.floatbank10_sulfate	secondary_cleaner.state.floatbank2_a_air	primary_cleaner.input.xanthate	primary_cleaner.input.depressant	rougher.input.feed_rate
count	19542.000000	19718.000000	19954.000000	19963.000000	19965.000000	20003.000000	20033.000000	20033.000000	20041.000000
mean	6.058794	1072.584818	12.069147	145.346889	12.314856	28.535827	1.013894	8.863749	473.234067
std	1.116986	185.794118	3.780202	44.634608	3.469103	5.850875	0.519693	3.345438	110.765504
min	0.000290	-1.970275	0.000049	0.003112	0.000044	0.025693	0.000006	0.000000	0.001166
25%	5.405188	997.822139	9.998294	114.747722	10.000173	25.047215	0.694630	6.095366	407.138551
50%	6.099767	1049.694499	12.000267	144.133440	12.001148	29.086048	0.942111	8.044753	498.480665
75%	6.801447	1199.174819	14.576829	175.825511	14.679636	32.987186	1.207878	11.030483	549.840741
max	9.698407	1922.636637	37.980648	265.983123	36.118275	60.000000	4.102454	20.673152	717.508837

	date	final.output.recovery	primary_cleaner.input.sulfate	primary_cleaner.input.depressant	primary_cleaner.input.feed_size	primary_cleaner.input.xanthate	primary_cleaner.state.floatbank8_a_air	primary_cleaner.state.floatbank8_a_level	primary_cleaner.state.floatbank8_b_air	primary_cleaner.state.floatbank8_b_level	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
2918	2016-05-15 13:59:59	79.975380	159.551841	11.972081	7.740000	0.904101	1203.830417	-499.751674	1204.874868	-499.591962	...	9.989485	-501.018930	8.028083	-504.302157	8.806407	-499.302177	5.980273	-500.507677	20.006564	-496.164292
2919	2016-05-15 14:59:59	45.586191	58.717323	12.027662	6.610000	0.922359	1392.770893	-500.326034	1299.437987	-499.958419	...	10.012410	-501.591840	7.907263	-513.814631	8.568602	-498.174475	5.968032	-499.963947	19.985684	-503.306448
3446	2016-06-06 13:59:59	46.375447	0.832935	NaN	6.800000	0.010702	1191.181589	-502.140749	1192.766017	-494.615276	...	10.016232	-591.105501	8.034319	-547.173732	7.957247	-495.834105	6.025972	-500.221367	18.015904	-495.628009
4455	2016-07-18 14:59:59	52.576792	62.830953	7.006670	8.210000	0.574743	1601.946035	-500.703455	1607.720913	-500.257229	...	15.012221	-401.688850	4.974202	-400.312056	10.051200	-403.969189	5.039952	-400.190704	23.012079	-501.597792
5660	2017-01-06 19:59:59	67.275195	NaN	0.416502	7.790000	NaN	1854.162080	-615.091268	2114.906758	-593.303764	...	0.000000	-530.197868	0.000000	-799.619081	-0.112612	-792.794538	0.646208	-515.392892	0.238511	-809.398668
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
16502	2018-08-03 13:59:59	100.000000	NaN	NaN	7.280000	NaN	1650.905789	-494.820004	1648.101136	-570.593484	...	22.024408	-495.583514	16.985811	-499.811230	17.012212	-504.164124	12.998162	-500.848297	17.994886	-618.618605
16598	2018-08-07 13:59:59	100.000000	0.089524	NaN	7.004999	0.004911	998.890138	-763.169284	1069.561022	-799.562139	...	19.778919	-691.281395	16.481060	-507.062871	16.710791	-752.526118	13.087867	-799.649084	16.549049	-776.686885
16599	2018-08-07 14:59:59	100.000000	0.017954	NaN	7.320000	0.012019	1602.086817	-583.850387	1870.308253	-667.950493	...	26.990625	-506.292971	23.023367	-499.713257	22.950561	-587.299584	17.938124	-640.422102	22.951411	-625.940102
16600	2018-08-07 15:59:59	100.000000	0.042164	0.020230	7.320000	0.003251	1579.144432	-483.492915	1845.758204	-505.822619	...	26.981932	-240.370737	22.948309	-161.049087	22.945266	-301.530549	17.774928	-321.616150	22.941181	-339.193550
16608	2018-08-07 23:59:59	100.000000	NaN	0.133502	7.320000	0.005070	1567.512693	-400.292034	1873.332954	-399.589371	...	26.975033	-502.882224	23.075104	-499.854192	22.984960	-539.538899	17.997212	-499.904563	22.987740	-625.612323

	rougher.input.floatbank11_xanthate	rougher.input.floatbank10_xanthate
2916	4.403525	4.403645
2917	3.911547	4.400047
2918	5.418245	5.418245
2919	4.744994	4.744994
2920	4.476183	4.755170
...	...	...
16855	9.156069	9.158609
16856	9.297924	9.304952
16857	9.300133	9.299606
16858	9.297194	9.297709
16859	9.309852	9.308612

	date	primary_cleaner.input.sulfate	primary_cleaner.input.depressant	primary_cleaner.input.feed_size	primary_cleaner.input.xanthate	primary_cleaner.state.floatbank8_a_air	primary_cleaner.state.floatbank8_a_level	primary_cleaner.state.floatbank8_b_air	primary_cleaner.state.floatbank8_b_level	primary_cleaner.state.floatbank8_c_air	...	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
245	2016-09-11 05:59:59	15.630879	0.990310	7.83	0.193422	1.299719e+03	-498.963997	1.303213e+03	-499.904044	1.301021e+03	...	8.021303	-506.306992	4.895868	-502.255707	7.904954	-508.638295	3.043868	-501.832155	19.989689	-603.877450
246	2016-09-11 06:59:59	NaN	NaN	7.83	NaN	1.299738e+03	-612.606492	1.302267e+03	-505.831097	1.299494e+03	...	8.073119	-501.810437	4.956902	-499.412118	8.153360	-492.626261	3.081493	-476.885347	20.035238	-603.205354
247	2016-09-11 07:59:59	NaN	NaN	7.83	NaN	1.300137e+03	-752.693115	1.302233e+03	-575.075180	1.299606e+03	...	8.022703	-520.809679	4.954530	-569.697771	7.900118	-513.604732	2.960581	-499.693953	19.998444	-734.132064
248	2016-09-11 08:59:59	NaN	NaN	7.83	NaN	1.297507e+03	-794.938526	1.303108e+03	-798.079260	1.300093e+03	...	7.907092	-509.244830	4.972887	-580.824925	8.079996	-519.416398	2.948391	-499.492902	19.993156	-749.453971
249	2016-09-11 09:59:59	NaN	NaN	7.83	NaN	1.296324e+03	-795.541101	1.302574e+03	-798.501004	1.299455e+03	...	8.030514	-517.587841	5.054695	-668.750178	8.218528	-552.623825	2.967558	-499.669186	20.015216	-798.761749
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
5728	2017-12-26 16:59:59	68.748269	4.967088	5.80	0.546002	1.601695e+03	-401.309128	1.603799e+03	-398.454978	1.306149e+03	...	20.007729	-500.282320	15.116499	-500.088520	11.053275	-500.223931	8.996529	-499.892002	12.003847	-500.072328
5729	2017-12-26 17:59:59	66.671316	4.982771	5.80	0.546054	1.601798e+03	-399.342516	1.601004e+03	-399.589165	1.309597e+03	...	19.970385	-499.739051	15.034725	-499.893765	11.005241	-498.989282	9.046014	-499.756903	11.983336	-498.917245
5747	2017-12-27 11:59:59	7.469704	NaN	7.20	0.004984	5.445860e-32	-782.195107	6.647490e-32	-496.247779	4.033736e-32	...	3.265143	-799.427634	1.080343	-799.879409	0.200600	-797.323986	1.420606	-800.118587	0.016815	-789.840007
5748	2017-12-27 12:59:59	5.630580	NaN	7.20	NaN	9.261879e+02	-686.329732	9.278570e+02	-548.715931	8.940153e+02	...	5.882153	-798.499014	3.409673	-799.002255	2.406223	-797.219304	2.938882	-800.082169	2.454536	-803.594266
5749	2017-12-27 13:59:59	2.620827	NaN	7.20	0.004011	1.394345e+03	-771.082673	1.394144e+03	-777.498223	1.398281e+03	...	19.962331	-797.550678	14.999044	-798.207155	10.961253	-797.003879	8.987172	-800.059438	12.016949	-804.138159

	date	primary_cleaner.input.sulfate	primary_cleaner.input.depressant	primary_cleaner.input.feed_size	primary_cleaner.input.xanthate	primary_cleaner.state.floatbank8_a_air	primary_cleaner.state.floatbank8_a_level	primary_cleaner.state.floatbank8_b_air	primary_cleaner.state.floatbank8_b_level	primary_cleaner.state.floatbank8_c_air	primary_cleaner.state.floatbank8_c_level	primary_cleaner.state.floatbank8_d_air	primary_cleaner.state.floatbank8_d_level	rougher.input.feed_ag	rougher.input.feed_pb	rougher.input.feed_rate	rougher.input.feed_size	rougher.input.feed_sol	rougher.input.feed_au	rougher.input.floatbank10_sulfate	rougher.input.floatbank10_xanthate	rougher.input.floatbank11_sulfate	rougher.input.floatbank11_xanthate	rougher.state.floatbank10_a_air	rougher.state.floatbank10_a_level	rougher.state.floatbank10_b_air	rougher.state.floatbank10_b_level	rougher.state.floatbank10_c_air	rougher.state.floatbank10_c_level	rougher.state.floatbank10_d_air	rougher.state.floatbank10_d_level	rougher.state.floatbank10_e_air	rougher.state.floatbank10_e_level	rougher.state.floatbank10_f_air	rougher.state.floatbank10_f_level	secondary_cleaner.state.floatbank2_a_air	secondary_cleaner.state.floatbank2_a_level	secondary_cleaner.state.floatbank2_b_air	secondary_cleaner.state.floatbank2_b_level	secondary_cleaner.state.floatbank3_a_air	secondary_cleaner.state.floatbank3_a_level	secondary_cleaner.state.floatbank3_b_air	secondary_cleaner.state.floatbank3_b_level	secondary_cleaner.state.floatbank4_a_air	secondary_cleaner.state.floatbank4_a_level	secondary_cleaner.state.floatbank4_b_air	secondary_cleaner.state.floatbank4_b_level	secondary_cleaner.state.floatbank5_a_air	secondary_cleaner.state.floatbank5_a_level	secondary_cleaner.state.floatbank5_b_air	secondary_cleaner.state.floatbank5_b_level	secondary_cleaner.state.floatbank6_a_air	secondary_cleaner.state.floatbank6_a_level
564	2016-09-24 12:59:59	NaN	NaN	7.68	0.011503	2.799954e-18	-796.779434	4.163275e-21	-799.983618	3.855577e-19	-799.948108	2.014029e-21	-799.784812	0.000000	0.000000	0.193756	48.015584	0.0	0.000000	NaN	NaN	0.096795	NaN	0.001266	-650.096515	-0.605473	-650.105302	-0.068244	-645.931153	-0.924575	-628.468917	-1.944313	-650.122621	-2.381344	-649.913483	0.023057	-799.530262	1.092967e-16	-792.875062	1.136677e-18	-799.561997	1.807957e-20	-734.556570	1.834961e-22	-745.821862	6.161089e-18	-799.802846	-0.214178	-797.367106	0.704683	-800.020932	0.161888	-799.913285
565	2016-09-24 13:59:59	NaN	NaN	7.68	0.000091	2.799954e-18	-797.124505	4.163275e-21	-799.977005	3.855577e-19	-799.950312	2.014029e-21	-799.771693	0.000000	0.000000	0.204528	48.018385	0.0	0.000000	NaN	NaN	0.105139	NaN	0.002585	-650.069770	-0.650662	-650.073165	-0.071626	-644.887999	-0.966635	-629.221695	-1.921022	-650.134710	-2.404892	-649.888636	0.008992	-799.545795	1.092967e-16	-794.451180	1.136677e-18	-799.586654	1.807957e-20	-735.627312	1.834961e-22	-746.906048	6.161089e-18	-799.797453	-0.215287	-798.454745	0.711425	-800.021343	0.157048	-799.922919
568	2016-09-24 16:59:59	NaN	NaN	7.68	0.003025	1.392885e-01	-798.855473	5.527349e-02	-799.989777	1.014484e+00	-799.981199	9.770101e-01	-799.827218	0.000000	0.000000	0.212210	48.026788	0.0	0.000000	NaN	NaN	0.021897	NaN	-0.001964	-650.056702	-0.685651	-650.091352	-0.062618	-645.245733	-1.018060	-636.253339	-1.881713	-650.142921	-2.436634	-649.857932	0.010803	-799.495730	0.000000e+00	-794.478846	0.000000e+00	-799.555332	0.000000e+00	-738.153816	0.000000e+00	-748.585027	0.000000e+00	-799.831149	-0.186699	-797.669733	0.722592	-799.997136	0.179766	-799.936573
585	2016-09-25 09:59:59	NaN	NaN	7.68	0.010103	2.312186e-20	-796.302283	5.786564e-21	-799.984843	1.205502e-20	-799.962190	3.571137e-21	-799.793338	0.000000	0.000000	0.166748	48.033980	0.0	0.000000	NaN	NaN	0.030064	NaN	-0.087418	-650.165051	-0.631677	-650.180366	-0.230563	-642.077940	-0.831368	-623.281513	-1.963070	-650.037184	-2.356340	-649.943086	3.935541	-799.491407	0.000000e+00	-791.800662	0.000000e+00	-799.609465	0.000000e+00	-732.983100	0.000000e+00	-750.653885	0.000000e+00	-747.383683	-0.181728	-796.941962	0.690744	-800.028909	0.202145	-799.903983
586	2016-09-25 10:59:59	NaN	NaN	7.68	0.006077	2.312186e-20	-796.684510	5.786564e-21	-799.981251	1.205502e-20	-799.964183	3.571137e-21	-799.799478	0.000000	0.000000	0.178374	48.034314	0.0	0.000000	NaN	NaN	0.109840	NaN	-0.068932	-650.134236	-0.669726	-650.147578	-0.240868	-647.688869	-0.846947	-628.388343	-1.951958	-650.056851	-2.370610	-649.918108	0.475950	-799.497027	0.000000e+00	-792.206187	0.000000e+00	-799.613829	0.000000e+00	-734.963840	0.000000e+00	-750.157445	0.000000e+00	-747.406024	-0.197555	-797.115802	0.691613	-800.026245	0.187614	-799.911700
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3078	2017-09-07 06:59:59	NaN	NaN	6.44	NaN	0.000000e+00	-797.408631	0.000000e+00	-799.709651	0.000000e+00	-799.952526	0.000000e+00	-799.761447	0.000000	0.000000	1.747975	51.061615	0.0	0.000000	NaN	NaN	0.006781	NaN	-0.030619	-651.731882	-0.298349	-650.280348	-0.026257	-634.300938	-0.355675	-627.073357	-1.843096	-641.492751	-2.154654	-649.953740	5.457716	-799.541372	0.000000e+00	-778.422522	0.000000e+00	-799.610773	0.000000e+00	-666.111484	0.000000e+00	-799.734266	0.000000e+00	-799.861995	0.501476	-798.069070	0.657183	-800.035928	0.291482	-809.551876
3079	2017-09-07 07:59:59	NaN	NaN	6.44	NaN	0.000000e+00	-797.596816	0.000000e+00	-799.189274	0.000000e+00	-799.953647	0.000000e+00	-799.765142	0.000000	0.000000	1.744616	51.064115	0.0	0.000000	NaN	NaN	0.007444	NaN	-0.019793	-651.208723	-0.328038	-650.261088	-0.022318	-639.661897	-0.402197	-623.377953	-1.847239	-642.092749	-2.201130	-649.943623	5.445182	-799.542015	0.000000e+00	-778.430384	0.000000e+00	-799.606907	0.000000e+00	-665.891486	0.000000e+00	-799.730764	0.000000e+00	-799.864521	0.507308	-798.682121	0.650173	-800.038423	0.295164	-809.087251
4549	2017-11-07 13:59:59	0.006246	NaN	7.15	0.001874	1.601321e+03	-506.241798	1.598187e+03	-503.990762	1.496591e+03	-496.429140	1.598291e+03	-502.147076	0.295709	0.148351	5.544637	64.104260	NaN	0.302262	NaN	NaN	0.041485	NaN	1195.228167	-300.289581	1396.458446	-499.850158	1345.411732	-500.639176	1243.202842	-501.887101	1148.085706	-500.526577	1048.310270	-498.953279	0.838440	-773.427423	3.603394e-03	-795.667200	2.904193e+01	-500.249506	2.300451e+01	-500.010143	2.101980e+01	-502.283743	1.901142e+01	-500.743581	15.057985	-499.898364	7.928971	-499.832515	16.020571	-499.911516
5485	2017-12-16 13:59:59	NaN	NaN	7.49	0.012088	1.198172e+00	-783.388562	2.241474e+00	-799.970691	1.687280e+00	-799.644774	1.665990e+00	-579.911423	0.000000	0.000000	NaN	25.351381	0.0	0.000000	NaN	NaN	0.010526	NaN	999.321327	-413.098504	1200.388407	-650.012364	1198.360990	-500.612020	999.776180	-617.944162	999.757989	-500.139406	900.204461	-648.189008	25.074803	-753.629855	1.999943e+01	-506.324089	2.498809e+01	-798.731928	1.804578e+01	-809.149110	7.349942e-17	-799.649342	8.306354e-17	-799.873377	10.949957	-608.576945	7.972650	-369.027573	12.000541	-804.865410
5486	2017-12-16 14:59:59	NaN	NaN	7.49	0.008673	5.547224e-01	-784.049738	1.035048e+00	-799.985963	7.791375e-01	-799.864020	7.693061e-01	-581.016175	0.000000	0.000000	NaN	25.352417	0.0	0.000000	NaN	NaN	0.029582	NaN	1001.081732	-412.607977	1200.722607	-650.016866	1201.885233	-503.141458	1001.078637	-622.168851	1001.112081	-499.743099	901.047630	-647.264918	25.094838	-789.752004	1.991080e+01	-508.635696	2.494250e+01	-798.892687	1.801429e+01	-809.148189	7.349942e-17	-799.649104	8.306354e-17	-799.877852	10.909520	-639.780856	7.952206	-354.341867	11.982311	-804.065385

Statistic	floatbank10_xanthate	floatbank11_xanthate	xanthate_difference
Median	6.243	7.494	-1.416
Mean	4.438	6.770	-2.333
Range	7.6248, 0.001	8.834, 1.482	-1.001, -8.005

Statistic	floatbank10_xanthate	floatbank11_xanthate	xanthate_difference
Median	5.781	1.931	3.204
Mean	5.703	2.413	3.290
Range	8.036, 3.109	6.318, 0.000	7.576, 1.026

Statistic	floatbank10_xanthate	floatbank11_xanthate	xanthate_difference
Median	6.000	5.998	0.002
Mean	5.908	5.898	0.010
Range	9.703, 0.004	9.698, 0.002	0.953, 0.000

Statistic	floatbank10_xanthate	floatbank11_xanthate	xanthate_difference
Median	5.995	6.000	-0.002
Mean	5.858	5.902	-0.044
Range	9.655, 0.001	9.667, 0.001	-0.000, -1.000